import functools

import numpy as np
import tensorflow as tf
import scipy
import scipy.linalg
import matplotlib.pyplot as plt

from tensorflow import keras
from tensorflow.keras import layers
import tensorflow.keras.backend as K

from sklearn.model_selection import train_test_split

reg = keras.regularizers.l1_l2(l1=1e-5, l2=1e-5)
#reg = None

regularize = lambda f: functools.partial(f,
                           kernel_regularizer=reg,
                           bias_regularizer=reg,
                           kernel_initializer='he_normal',
                           bias_initializer='he_normal',)

Conv2D = regularize(layers.Conv2D)
Dense = regularize(layers.Dense)

# Load data.
images = np.load("images.npy")
labels = np.load("labels.npy")
print(images.shape, labels.shape)

(18000, 150, 150) (18000, 2)

# Preprocess data.
X = images.reshape(images.shape + (1,))

# Translate labels into one-hot encoding.
n = labels.shape[0]
hour_labels = labels[:,0]
minute_labels = labels[:,1]

# We actually encode half-hours to disambiguate 59/0 minutes later.
y_real = hour_labels * 60 + minute_labels
y_hours = np.zeros((n, 24))
y_hours[np.arange(n), 2 * y_real // 60] = 1
y_minutes = np.zeros((n, 60))
y_minutes[np.arange(n), minute_labels] = 1
y = np.column_stack([y_hours, y_minutes])

def split_hours_minutes(y):
    return y[...,:24], y[...,24:]

# Reverse transform.
def decode_onehot(y, lib=np):
    y_hours, y_minutes = split_hours_minutes(y)
    half_hours = lib.argmax(y_hours, axis=1)
    minutes = lib.argmax(y_minutes, axis=1)
    hours = half_hours // 2
    second_half_hour = (half_hours - 2*hours) == 1
    if lib is np:
        hours += second_half_hour & (minutes <= 15)
        hours -= ~second_half_hour & (minutes >= 45)
    else:
        hours += lib.cast(second_half_hour & (minutes <= 15), 'int64')
        hours -= lib.cast(~second_half_hour & (minutes >= 45), 'int64')
    return (hours * 60 + minutes) % (12*60)

(X_train, X_test,
 y_train, y_test,
 y_real_train, y_real_test) = train_test_split(X, y, y_real, test_size=0.2, random_state=42)

def cyclic_error_minutes(true, pred, lib=np):
    M = 12*60
    err = lib.abs(true - pred) % M
    return lib.minimum(err, M - err)

# Define neural network architecture.
def norm_act_conv(x, filters, kernel_size, strides=(1, 1), activation=True, padding='same'):
    x = layers.BatchNormalization()(x)
    if activation: x = layers.ReLU()(x)
    x = Conv2D(filters, kernel_size, strides=strides, padding=padding)(x)
    return x

def residual_bottleneck_block_v2(input_tensor, filters, half_resolution):
    input_filters = input_tensor.shape[-1]
    downscale_strides = (2, 2) if half_resolution else (1, 1)
    
    y = input_tensor
    x = norm_act_conv(input_tensor,   filters, (1, 1))
    x = norm_act_conv(x,              filters, (3, 3), strides=downscale_strides)
    x = norm_act_conv(x,            4*filters, (1, 1))
    
    if x.shape[1:] != y.shape[1:]:
        y = Conv2D(4*filters, (1, 1), strides=downscale_strides)(y)

    return layers.ReLU()(layers.add([x, y]))

def resnet50_v2(input_tensor):
    x = layers.ZeroPadding2D(padding=(3, 3))(input_tensor)
    x = Conv2D(64, (7, 7),  strides=(2, 2), padding='valid')(x)
    x = layers.BatchNormalization()(x)
    x = layers.ReLU()(x)
    x = layers.ZeroPadding2D(padding=(1, 1))(x)
    x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='valid')(x)
    for i in range(3): x = residual_bottleneck_block_v2(x, 64, False)
    for i in range(4): x = residual_bottleneck_block_v2(x, 128, i == 0)
    for i in range(6): x = residual_bottleneck_block_v2(x, 256, i == 0)
    for i in range(3): x = residual_bottleneck_block_v2(x, 512, i == 0)
    x = layers.BatchNormalization()(x)
    x = layers.ReLU()(x)
    return x

def clocknet_classify(input_tensor):
    x = resnet50_v2(input_tensor)
    x = layers.GlobalAveragePooling2D()(x)
    x = Dense(1000)(x)
    x = layers.BatchNormalization()(x)
    resnet_out = layers.ReLU()(x)
    hours = Dense(24, name='hours_out', activation='softmax', kernel_regularizer=None, bias_regularizer=None)(resnet_out)
    minutes = Dense(60, name='minutes_out', activation='softmax', kernel_regularizer=None, bias_regularizer=None)(resnet_out)
    return hours, minutes

def cyclic_weighted_cross_entropy(n, weight):
    cce = keras.losses.CategoricalCrossentropy()
    circ = scipy.linalg.circulant(np.arange(n)).T
    cost_mat = np.minimum(circ, n - circ).astype('float32')
    
    def loss_func(y_true, y_pred):        
        y_pred_max_mat = K.one_hot(K.argmax(y_pred), num_classes=y_pred.shape[1])
        actual_error = K.sum((y_true @ cost_mat) * y_pred_max_mat, -1)
        expected_error = K.sum((y_true @ cost_mat) * y_pred, -1)
        sample_weight = 1 + expected_error * weight
        return K.mean(K.categorical_crossentropy(y_true, y_pred) * sample_weight)
    
    return loss_func

def time_loss():
    hour_loss = cyclic_weighted_cross_entropy(24, 30)
    minute_loss = cyclic_weighted_cross_entropy(60, 1)
    def loss(y_true, y_pred):
        y_true_hours, y_true_minutes = split_hours_minutes(y_true)
        y_pred_hours, y_pred_minutes = split_hours_minutes(y_pred)
        return hour_loss(y_true_hours, y_pred_hours) + minute_loss(y_true_minutes, y_pred_minutes)
    return loss

def hour_accuracy(y_true, y_pred):
    y_true_hours, y_true_minutes = split_hours_minutes(y_true)
    y_pred_hours, y_pred_minutes = split_hours_minutes(y_pred)
    return keras.metrics.categorical_accuracy(y_true_hours, y_pred_hours)

def minute_accuracy(y_true, y_pred):
    y_true_hours, y_true_minutes = split_hours_minutes(y_true)
    y_pred_hours, y_pred_minutes = split_hours_minutes(y_pred)
    return keras.metrics.categorical_accuracy(y_true_minutes, y_pred_minutes)

def real_time_error(y_true, y_pred):
    true_time = K.cast(decode_onehot(y_true, lib=K), 'float64')
    pred_time = K.cast(decode_onehot(y_pred, lib=K), 'float64')
    return cyclic_error_minutes(true_time, pred_time, lib=K)

def create_model():
    inp = layers.Input((150, 150, 1))
    cnh, cnm = clocknet_classify(inp)
    out = layers.concatenate([cnh, cnm], name='time')
    model = keras.Model(inputs=inp, outputs=out)
    return model

model = create_model()

filepath = "D:/study/models4/model-adam-{epoch:02d}.hdf5"
csv_logger = keras.callbacks.CSVLogger('adam_training.log')
checkpoint = keras.callbacks.ModelCheckpoint(filepath, verbose=1, save_best_only=False)

model.compile(loss=time_loss(), optimizer='adam', metrics=[hour_accuracy, minute_accuracy, real_time_error])
hist1 = model.fit(X_train,
                  y_train,
                  validation_data=(X_test, y_test),
                  callbacks=[csv_logger, checkpoint],
                  batch_size=24, verbose=True, epochs=100)


filepath = "D:/study/models4/model-sgd-{epoch:02d}.hdf5"
csv_logger = keras.callbacks.CSVLogger('sgd_training.log')
checkpoint = keras.callbacks.ModelCheckpoint(filepath, verbose=1, save_best_only=False)
lr_scheduler = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-4 * 0.95**epoch)
sgd = keras.optimizers.SGD(learning_rate=1e-4, momentum=0.9)
model.compile(loss=time_loss(), optimizer=sgd, metrics=[hour_accuracy, minute_accuracy, real_time_error])
hist2 = model.fit(X_train,
                  y_train,
                  validation_data=(X_test, y_test),
                  callbacks=[csv_logger, checkpoint, lr_scheduler],
                  batch_size=24, verbose=True, epochs=100)

Train on 14400 samples, validate on 3600 samples
Epoch 1/100
14376/14400 [============================>.] - ETA: 0s - loss: 487.0463 - hour_accuracy: 0.1463 - minute_accuracy: 0.0290 - real_time_error: 121.6997
Epoch 00001: saving model to D:/study/models4/model-adam-01.hdf5
14400/14400 [==============================] - 123s 9ms/sample - loss: 486.7834 - hour_accuracy: 0.1462 - minute_accuracy: 0.0290 - real_time_error: 121.6648 - val_loss: 1754.4505 - val_hour_accuracy: 0.0622 - val_minute_accuracy: 0.0219 - val_real_time_error: 139.0205
Epoch 2/100
14376/14400 [============================>.] - ETA: 0s - loss: 207.5141 - hour_accuracy: 0.3949 - minute_accuracy: 0.0484 - real_time_error: 56.2084
Epoch 00002: saving model to D:/study/models4/model-adam-02.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 207.3569 - hour_accuracy: 0.3951 - minute_accuracy: 0.0484 - real_time_error: 56.1636 - val_loss: 783.0214 - val_hour_accuracy: 0.1653 - val_minute_accuracy: 0.0178 - val_real_time_error: 73.4564
Epoch 3/100
14376/14400 [============================>.] - ETA: 0s - loss: 100.7091 - hour_accuracy: 0.5977 - minute_accuracy: 0.0874 - real_time_error: 28.1318
Epoch 00003: saving model to D:/study/models4/model-adam-03.hdf5
14400/14400 [==============================] - 108s 8ms/sample - loss: 100.7474 - hour_accuracy: 0.5973 - minute_accuracy: 0.0873 - real_time_error: 28.1526 - val_loss: 810.3652 - val_hour_accuracy: 0.1681 - val_minute_accuracy: 0.0197 - val_real_time_error: 90.4470
Epoch 4/100
14376/14400 [============================>.] - ETA: 0s - loss: 69.3074 - hour_accuracy: 0.7046 - minute_accuracy: 0.1152 - real_time_error: 19.3463
Epoch 00004: saving model to D:/study/models4/model-adam-04.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 69.3512 - hour_accuracy: 0.7047 - minute_accuracy: 0.1151 - real_time_error: 19.3509 - val_loss: 539.2764 - val_hour_accuracy: 0.2331 - val_minute_accuracy: 0.0272 - val_real_time_error: 83.8497
Epoch 5/100
14376/14400 [============================>.] - ETA: 0s - loss: 47.3971 - hour_accuracy: 0.8014 - minute_accuracy: 0.1536 - real_time_error: 12.4503
Epoch 00005: saving model to D:/study/models4/model-adam-05.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 47.4098 - hour_accuracy: 0.8012 - minute_accuracy: 0.1534 - real_time_error: 12.4576 - val_loss: 179.0487 - val_hour_accuracy: 0.3811 - val_minute_accuracy: 0.0589 - val_real_time_error: 40.2700
Epoch 6/100
14376/14400 [============================>.] - ETA: 0s - loss: 36.2834 - hour_accuracy: 0.8582 - minute_accuracy: 0.1762 - real_time_error: 8.5548
Epoch 00006: saving model to D:/study/models4/model-adam-06.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 36.2748 - hour_accuracy: 0.8582 - minute_accuracy: 0.1762 - real_time_error: 8.5560 - val_loss: 29.3075 - val_hour_accuracy: 0.8831 - val_minute_accuracy: 0.1344 - val_real_time_error: 6.1150
Epoch 7/100
14376/14400 [============================>.] - ETA: 0s - loss: 37.9664 - hour_accuracy: 0.8799 - minute_accuracy: 0.2128 - real_time_error: 7.8819
Epoch 00007: saving model to D:/study/models4/model-adam-07.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 38.0646 - hour_accuracy: 0.8794 - minute_accuracy: 0.2129 - real_time_error: 7.9165 - val_loss: 2133.8827 - val_hour_accuracy: 0.0683 - val_minute_accuracy: 0.0197 - val_real_time_error: 158.3639
Epoch 8/100
14376/14400 [============================>.] - ETA: 0s - loss: 44.9045 - hour_accuracy: 0.8171 - minute_accuracy: 0.1761 - real_time_error: 11.4125
Epoch 00008: saving model to D:/study/models4/model-adam-08.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 44.8792 - hour_accuracy: 0.8171 - minute_accuracy: 0.1760 - real_time_error: 11.3999 - val_loss: 32.7908 - val_hour_accuracy: 0.8728 - val_minute_accuracy: 0.1736 - val_real_time_error: 7.3650
Epoch 9/100
14376/14400 [============================>.] - ETA: 0s - loss: 23.2195 - hour_accuracy: 0.9307 - minute_accuracy: 0.2384 - real_time_error: 3.7168
Epoch 00009: saving model to D:/study/models4/model-adam-09.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 23.2191 - hour_accuracy: 0.9307 - minute_accuracy: 0.2384 - real_time_error: 3.7204 - val_loss: 89.6355 - val_hour_accuracy: 0.6553 - val_minute_accuracy: 0.1211 - val_real_time_error: 18.1458
Epoch 10/100
14376/14400 [============================>.] - ETA: 0s - loss: 41.1812 - hour_accuracy: 0.8742 - minute_accuracy: 0.2326 - real_time_error: 8.4981
Epoch 00010: saving model to D:/study/models4/model-adam-10.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 41.2488 - hour_accuracy: 0.8736 - minute_accuracy: 0.2324 - real_time_error: 8.5314 - val_loss: 3133.1379 - val_hour_accuracy: 0.0419 - val_minute_accuracy: 0.0161 - val_real_time_error: 179.6464
Epoch 11/100
14376/14400 [============================>.] - ETA: 0s - loss: 42.0385 - hour_accuracy: 0.8486 - minute_accuracy: 0.2042 - real_time_error: 9.8476
Epoch 00011: saving model to D:/study/models4/model-adam-11.hdf5
14400/14400 [==============================] - 108s 8ms/sample - loss: 42.0008 - hour_accuracy: 0.8488 - minute_accuracy: 0.2041 - real_time_error: 9.8333 - val_loss: 59.8073 - val_hour_accuracy: 0.7667 - val_minute_accuracy: 0.1492 - val_real_time_error: 15.6275
Epoch 12/100
14376/14400 [============================>.] - ETA: 0s - loss: 19.6353 - hour_accuracy: 0.9608 - minute_accuracy: 0.2916 - real_time_error: 2.2940
Epoch 00012: saving model to D:/study/models4/model-adam-12.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 19.6436 - hour_accuracy: 0.9608 - minute_accuracy: 0.2915 - real_time_error: 2.2918 - val_loss: 19.7090 - val_hour_accuracy: 0.9558 - val_minute_accuracy: 0.2519 - val_real_time_error: 2.1447
Epoch 13/100
14376/14400 [============================>.] - ETA: 0s - loss: 33.6337 - hour_accuracy: 0.9096 - minute_accuracy: 0.2803 - real_time_error: 6.2529
Epoch 00013: saving model to D:/study/models4/model-adam-13.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 33.8494 - hour_accuracy: 0.9088 - minute_accuracy: 0.2800 - real_time_error: 6.2992 - val_loss: 2573.5840 - val_hour_accuracy: 0.0514 - val_minute_accuracy: 0.0169 - val_real_time_error: 173.8156
Epoch 14/100
14376/14400 [============================>.] - ETA: 0s - loss: 32.4662 - hour_accuracy: 0.9009 - minute_accuracy: 0.2453 - real_time_error: 6.0078
Epoch 00014: saving model to D:/study/models4/model-adam-14.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 32.4381 - hour_accuracy: 0.9010 - minute_accuracy: 0.2458 - real_time_error: 5.9990 - val_loss: 19.7485 - val_hour_accuracy: 0.9614 - val_minute_accuracy: 0.2703 - val_real_time_error: 2.0519
Epoch 15/100
14376/14400 [============================>.] - ETA: 0s - loss: 19.1233 - hour_accuracy: 0.9643 - minute_accuracy: 0.3177 - real_time_error: 1.9763
Epoch 00015: saving model to D:/study/models4/model-adam-15.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 19.1205 - hour_accuracy: 0.9644 - minute_accuracy: 0.3177 - real_time_error: 1.9746 - val_loss: 529.9782 - val_hour_accuracy: 0.2858 - val_minute_accuracy: 0.0433 - val_real_time_error: 88.4016
Epoch 16/100
14376/14400 [============================>.] - ETA: 0s - loss: 18.4689 - hour_accuracy: 0.9626 - minute_accuracy: 0.3274 - real_time_error: 2.1091
Epoch 00016: saving model to D:/study/models4/model-adam-16.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 18.4644 - hour_accuracy: 0.9626 - minute_accuracy: 0.3275 - real_time_error: 2.1073 - val_loss: 100.6778 - val_hour_accuracy: 0.7328 - val_minute_accuracy: 0.1736 - val_real_time_error: 23.6500
Epoch 17/100
14376/14400 [============================>.] - ETA: 0s - loss: 42.5465 - hour_accuracy: 0.8717 - minute_accuracy: 0.2737 - real_time_error: 8.9010
Epoch 00017: saving model to D:/study/models4/model-adam-17.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 42.5153 - hour_accuracy: 0.8718 - minute_accuracy: 0.2737 - real_time_error: 8.8922 - val_loss: 258.5011 - val_hour_accuracy: 0.4439 - val_minute_accuracy: 0.0808 - val_real_time_error: 44.0561
Epoch 18/100
14376/14400 [============================>.] - ETA: 0s - loss: 21.0206 - hour_accuracy: 0.9578 - minute_accuracy: 0.3080 - real_time_error: 2.3763
Epoch 00018: saving model to D:/study/models4/model-adam-18.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 21.0228 - hour_accuracy: 0.9578 - minute_accuracy: 0.3079 - real_time_error: 2.3742 - val_loss: 939.1564 - val_hour_accuracy: 0.1761 - val_minute_accuracy: 0.0528 - val_real_time_error: 115.0258
Epoch 19/100
14376/14400 [============================>.] - ETA: 0s - loss: 29.6825 - hour_accuracy: 0.9215 - minute_accuracy: 0.2992 - real_time_error: 5.3758
Epoch 00019: saving model to D:/study/models4/model-adam-19.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 29.6887 - hour_accuracy: 0.9213 - minute_accuracy: 0.2992 - real_time_error: 5.3767 - val_loss: 1951.5232 - val_hour_accuracy: 0.0697 - val_minute_accuracy: 0.0189 - val_real_time_error: 143.4945
Epoch 20/100
14376/14400 [============================>.] - ETA: 0s - loss: 23.6692 - hour_accuracy: 0.9486 - minute_accuracy: 0.3122 - real_time_error: 3.0938
Epoch 00020: saving model to D:/study/models4/model-adam-20.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 23.6574 - hour_accuracy: 0.9487 - minute_accuracy: 0.3124 - real_time_error: 3.0898 - val_loss: 19.3507 - val_hour_accuracy: 0.9689 - val_minute_accuracy: 0.3053 - val_real_time_error: 1.5700
Epoch 21/100
14376/14400 [============================>.] - ETA: 0s - loss: 16.8900 - hour_accuracy: 0.9830 - minute_accuracy: 0.3962 - real_time_error: 1.1140
Epoch 00021: saving model to D:/study/models4/model-adam-21.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 16.8913 - hour_accuracy: 0.9829 - minute_accuracy: 0.3958 - real_time_error: 1.1141 - val_loss: 18.7178 - val_hour_accuracy: 0.9592 - val_minute_accuracy: 0.3558 - val_real_time_error: 1.9747
Epoch 22/100
14376/14400 [============================>.] - ETA: 0s - loss: 31.5129 - hour_accuracy: 0.9261 - minute_accuracy: 0.3513 - real_time_error: 5.2286
Epoch 00022: saving model to D:/study/models4/model-adam-22.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 31.5108 - hour_accuracy: 0.9260 - minute_accuracy: 0.3515 - real_time_error: 5.2219 - val_loss: 1625.5261 - val_hour_accuracy: 0.1339 - val_minute_accuracy: 0.0417 - val_real_time_error: 127.1422
Epoch 23/100
14376/14400 [============================>.] - ETA: 0s - loss: 21.1279 - hour_accuracy: 0.9606 - minute_accuracy: 0.3356 - real_time_error: 2.1917
Epoch 00023: saving model to D:/study/models4/model-adam-23.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 21.1195 - hour_accuracy: 0.9606 - minute_accuracy: 0.3358 - real_time_error: 2.1893 - val_loss: 401.9804 - val_hour_accuracy: 0.3914 - val_minute_accuracy: 0.0567 - val_real_time_error: 63.5628
Epoch 24/100
14376/14400 [============================>.] - ETA: 0s - loss: 17.3333 - hour_accuracy: 0.9795 - minute_accuracy: 0.3979 - real_time_error: 1.3347
Epoch 00024: saving model to D:/study/models4/model-adam-24.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 17.3312 - hour_accuracy: 0.9796 - minute_accuracy: 0.3978 - real_time_error: 1.3338 - val_loss: 21.9817 - val_hour_accuracy: 0.9508 - val_minute_accuracy: 0.3117 - val_real_time_error: 3.2156
Epoch 25/100
14376/14400 [============================>.] - ETA: 0s - loss: 25.8832 - hour_accuracy: 0.9418 - minute_accuracy: 0.3593 - real_time_error: 3.8148
Epoch 00025: saving model to D:/study/models4/model-adam-25.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 25.8763 - hour_accuracy: 0.9419 - minute_accuracy: 0.3591 - real_time_error: 3.8142 - val_loss: 1698.2786 - val_hour_accuracy: 0.1122 - val_minute_accuracy: 0.0158 - val_real_time_error: 128.1439
Epoch 26/100
14376/14400 [============================>.] - ETA: 0s - loss: 22.1671 - hour_accuracy: 0.9550 - minute_accuracy: 0.3539 - real_time_error: 2.7969
Epoch 00026: saving model to D:/study/models4/model-adam-26.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 22.1593 - hour_accuracy: 0.9551 - minute_accuracy: 0.3540 - real_time_error: 2.7933 - val_loss: 47.6067 - val_hour_accuracy: 0.8161 - val_minute_accuracy: 0.2178 - val_real_time_error: 8.2206
Epoch 27/100
14376/14400 [============================>.] - ETA: 0s - loss: 18.3953 - hour_accuracy: 0.9725 - minute_accuracy: 0.4035 - real_time_error: 1.6605
Epoch 00027: saving model to D:/study/models4/model-adam-27.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 18.3924 - hour_accuracy: 0.9725 - minute_accuracy: 0.4033 - real_time_error: 1.6590 - val_loss: 424.3513 - val_hour_accuracy: 0.4011 - val_minute_accuracy: 0.0469 - val_real_time_error: 62.0003
Epoch 28/100
14376/14400 [============================>.] - ETA: 0s - loss: 32.1012 - hour_accuracy: 0.9202 - minute_accuracy: 0.3260 - real_time_error: 5.4245
Epoch 00028: saving model to D:/study/models4/model-adam-28.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 32.0776 - hour_accuracy: 0.9203 - minute_accuracy: 0.3261 - real_time_error: 5.4171 - val_loss: 564.7458 - val_hour_accuracy: 0.2506 - val_minute_accuracy: 0.0753 - val_real_time_error: 70.6311
Epoch 29/100
14376/14400 [============================>.] - ETA: 0s - loss: 17.3940 - hour_accuracy: 0.9835 - minute_accuracy: 0.4202 - real_time_error: 1.0246
Epoch 00029: saving model to D:/study/models4/model-adam-29.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 17.3928 - hour_accuracy: 0.9835 - minute_accuracy: 0.4201 - real_time_error: 1.0242 - val_loss: 17.6880 - val_hour_accuracy: 0.9756 - val_minute_accuracy: 0.3878 - val_real_time_error: 1.2914
Epoch 30/100
14376/14400 [============================>.] - ETA: 0s - loss: 15.6074 - hour_accuracy: 0.9887 - minute_accuracy: 0.4641 - real_time_error: 0.7765
Epoch 00030: saving model to D:/study/models4/model-adam-30.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 15.6058 - hour_accuracy: 0.9887 - minute_accuracy: 0.4641 - real_time_error: 0.7762 - val_loss: 21.1975 - val_hour_accuracy: 0.9531 - val_minute_accuracy: 0.2808 - val_real_time_error: 2.7661
Epoch 31/100
14376/14400 [============================>.] - ETA: 0s - loss: 24.5827 - hour_accuracy: 0.9491 - minute_accuracy: 0.4067 - real_time_error: 3.7394
Epoch 00031: saving model to D:/study/models4/model-adam-31.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 24.5875 - hour_accuracy: 0.9490 - minute_accuracy: 0.4065 - real_time_error: 3.7390 - val_loss: 2349.7037 - val_hour_accuracy: 0.0683 - val_minute_accuracy: 0.0172 - val_real_time_error: 157.0242
Epoch 32/100
14376/14400 [============================>.] - ETA: 0s - loss: 25.0725 - hour_accuracy: 0.9520 - minute_accuracy: 0.3529 - real_time_error: 3.4597
Epoch 00032: saving model to D:/study/models4/model-adam-32.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 25.0589 - hour_accuracy: 0.9521 - minute_accuracy: 0.3528 - real_time_error: 3.4554 - val_loss: 959.3929 - val_hour_accuracy: 0.3014 - val_minute_accuracy: 0.1131 - val_real_time_error: 83.2783
Epoch 33/100
14376/14400 [============================>.] - ETA: 0s - loss: 16.5759 - hour_accuracy: 0.9861 - minute_accuracy: 0.4499 - real_time_error: 0.9405
Epoch 00033: saving model to D:/study/models4/model-adam-33.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 16.5741 - hour_accuracy: 0.9860 - minute_accuracy: 0.4499 - real_time_error: 0.9399 - val_loss: 20.8311 - val_hour_accuracy: 0.9542 - val_minute_accuracy: 0.2922 - val_real_time_error: 1.9022
Epoch 34/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.9421 - hour_accuracy: 0.9898 - minute_accuracy: 0.4818 - real_time_error: 0.7148
Epoch 00034: saving model to D:/study/models4/model-adam-34.hdf5
14400/14400 [==============================] - 108s 8ms/sample - loss: 14.9400 - hour_accuracy: 0.9899 - minute_accuracy: 0.4818 - real_time_error: 0.7144 - val_loss: 18.9286 - val_hour_accuracy: 0.9614 - val_minute_accuracy: 0.3853 - val_real_time_error: 1.9833
Epoch 35/100
14376/14400 [============================>.] - ETA: 0s - loss: 22.9864 - hour_accuracy: 0.9505 - minute_accuracy: 0.3913 - real_time_error: 3.4439
Epoch 00035: saving model to D:/study/models4/model-adam-35.hdf5
14400/14400 [==============================] - 108s 8ms/sample - loss: 23.0351 - hour_accuracy: 0.9503 - minute_accuracy: 0.3913 - real_time_error: 3.4603 - val_loss: 2480.2404 - val_hour_accuracy: 0.0725 - val_minute_accuracy: 0.0228 - val_real_time_error: 169.0653
Epoch 36/100
14376/14400 [============================>.] - ETA: 0s - loss: 20.4372 - hour_accuracy: 0.9672 - minute_accuracy: 0.4039 - real_time_error: 2.2042
Epoch 00036: saving model to D:/study/models4/model-adam-36.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 20.4314 - hour_accuracy: 0.9673 - minute_accuracy: 0.4040 - real_time_error: 2.2019 - val_loss: 22.2009 - val_hour_accuracy: 0.9581 - val_minute_accuracy: 0.2597 - val_real_time_error: 2.2844
Epoch 37/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.8647 - hour_accuracy: 0.9900 - minute_accuracy: 0.4875 - real_time_error: 0.7412
Epoch 00037: saving model to D:/study/models4/model-adam-37.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 14.8627 - hour_accuracy: 0.9900 - minute_accuracy: 0.4874 - real_time_error: 0.7410 - val_loss: 22.8123 - val_hour_accuracy: 0.9439 - val_minute_accuracy: 0.2269 - val_real_time_error: 3.1544
Epoch 38/100
14376/14400 [============================>.] - ETA: 0s - loss: 13.6002 - hour_accuracy: 0.9930 - minute_accuracy: 0.5243 - real_time_error: 0.6298
Epoch 00038: saving model to D:/study/models4/model-adam-38.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 13.5984 - hour_accuracy: 0.9930 - minute_accuracy: 0.5245 - real_time_error: 0.6294 - val_loss: 14.5361 - val_hour_accuracy: 0.9803 - val_minute_accuracy: 0.4322 - val_real_time_error: 0.9908
Epoch 39/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.1257 - hour_accuracy: 0.9858 - minute_accuracy: 0.4972 - real_time_error: 1.0243
Epoch 00039: saving model to D:/study/models4/model-adam-39.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 14.1317 - hour_accuracy: 0.9858 - minute_accuracy: 0.4970 - real_time_error: 1.0283 - val_loss: 3111.8695 - val_hour_accuracy: 0.0358 - val_minute_accuracy: 0.0117 - val_real_time_error: 187.7553
Epoch 40/100
14376/14400 [============================>.] - ETA: 0s - loss: 30.6526 - hour_accuracy: 0.9336 - minute_accuracy: 0.3653 - real_time_error: 4.9264
Epoch 00040: saving model to D:/study/models4/model-adam-40.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 30.6315 - hour_accuracy: 0.9337 - minute_accuracy: 0.3653 - real_time_error: 4.9195 - val_loss: 18.9765 - val_hour_accuracy: 0.9661 - val_minute_accuracy: 0.3072 - val_real_time_error: 1.9006
Epoch 41/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.9623 - hour_accuracy: 0.9905 - minute_accuracy: 0.4846 - real_time_error: 0.7898
Epoch 00041: saving model to D:/study/models4/model-adam-41.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 14.9610 - hour_accuracy: 0.9906 - minute_accuracy: 0.4844 - real_time_error: 0.7897 - val_loss: 16.6976 - val_hour_accuracy: 0.9708 - val_minute_accuracy: 0.3408 - val_real_time_error: 1.3322
Epoch 42/100
14376/14400 [============================>.] - ETA: 0s - loss: 13.7335 - hour_accuracy: 0.9924 - minute_accuracy: 0.5253 - real_time_error: 0.6664
Epoch 00042: saving model to D:/study/models4/model-adam-42.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 13.7320 - hour_accuracy: 0.9924 - minute_accuracy: 0.5254 - real_time_error: 0.6660 - val_loss: 14.7125 - val_hour_accuracy: 0.9814 - val_minute_accuracy: 0.3744 - val_real_time_error: 1.0286
Epoch 43/100
14376/14400 [============================>.] - ETA: 0s - loss: 13.9288 - hour_accuracy: 0.9855 - minute_accuracy: 0.5176 - real_time_error: 1.0264
Epoch 00043: saving model to D:/study/models4/model-adam-43.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 13.9276 - hour_accuracy: 0.9855 - minute_accuracy: 0.5173 - real_time_error: 1.0260 - val_loss: 242.0920 - val_hour_accuracy: 0.5122 - val_minute_accuracy: 0.0950 - val_real_time_error: 38.3128
Epoch 44/100
14376/14400 [============================>.] - ETA: 0s - loss: 23.6980 - hour_accuracy: 0.9512 - minute_accuracy: 0.4032 - real_time_error: 3.6810
Epoch 00044: saving model to D:/study/models4/model-adam-44.hdf5
14400/14400 [==============================] - 108s 7ms/sample - loss: 23.6816 - hour_accuracy: 0.9513 - minute_accuracy: 0.4031 - real_time_error: 3.6760 - val_loss: 15.3103 - val_hour_accuracy: 0.9803 - val_minute_accuracy: 0.4339 - val_real_time_error: 0.7039
Epoch 45/100
14376/14400 [============================>.] - ETA: 0s - loss: 13.9237 - hour_accuracy: 0.9919 - minute_accuracy: 0.5109 - real_time_error: 0.7139
Epoch 00045: saving model to D:/study/models4/model-adam-45.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 13.9221 - hour_accuracy: 0.9919 - minute_accuracy: 0.5110 - real_time_error: 0.7136 - val_loss: 13.6868 - val_hour_accuracy: 0.9886 - val_minute_accuracy: 0.5267 - val_real_time_error: 0.7331
Epoch 46/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.5354 - hour_accuracy: 0.9872 - minute_accuracy: 0.5196 - real_time_error: 1.0547
Epoch 00046: saving model to D:/study/models4/model-adam-46.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 14.5479 - hour_accuracy: 0.9872 - minute_accuracy: 0.5194 - real_time_error: 1.0585 - val_loss: 679.8856 - val_hour_accuracy: 0.2683 - val_minute_accuracy: 0.0506 - val_real_time_error: 87.0928
Epoch 47/100
14376/14400 [============================>.] - ETA: 0s - loss: 19.7415 - hour_accuracy: 0.9653 - minute_accuracy: 0.4352 - real_time_error: 2.3780
Epoch 00047: saving model to D:/study/models4/model-adam-47.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 19.7301 - hour_accuracy: 0.9653 - minute_accuracy: 0.4354 - real_time_error: 2.3748 - val_loss: 14.8327 - val_hour_accuracy: 0.9819 - val_minute_accuracy: 0.4331 - val_real_time_error: 0.8872
Epoch 48/100
14376/14400 [============================>.] - ETA: 0s - loss: 12.8561 - hour_accuracy: 0.9950 - minute_accuracy: 0.5510 - real_time_error: 0.6141
Epoch 00048: saving model to D:/study/models4/model-adam-48.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 12.8550 - hour_accuracy: 0.9950 - minute_accuracy: 0.5510 - real_time_error: 0.6138 - val_loss: 13.2647 - val_hour_accuracy: 0.9836 - val_minute_accuracy: 0.4883 - val_real_time_error: 0.7231
Epoch 49/100
14376/14400 [============================>.] - ETA: 0s - loss: 11.7957 - hour_accuracy: 0.9946 - minute_accuracy: 0.5864 - real_time_error: 0.5318
Epoch 00049: saving model to D:/study/models4/model-adam-49.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 11.7956 - hour_accuracy: 0.9946 - minute_accuracy: 0.5866 - real_time_error: 0.5315 - val_loss: 101.8926 - val_hour_accuracy: 0.6878 - val_minute_accuracy: 0.1242 - val_real_time_error: 23.6469
Epoch 50/100
14376/14400 [============================>.] - ETA: 0s - loss: 16.3512 - hour_accuracy: 0.9709 - minute_accuracy: 0.4875 - real_time_error: 1.9691
Epoch 00050: saving model to D:/study/models4/model-adam-50.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 16.3439 - hour_accuracy: 0.9710 - minute_accuracy: 0.4876 - real_time_error: 1.9667 - val_loss: 28.8856 - val_hour_accuracy: 0.9047 - val_minute_accuracy: 0.2742 - val_real_time_error: 5.3328
Epoch 51/100
14376/14400 [============================>.] - ETA: 0s - loss: 12.6003 - hour_accuracy: 0.9889 - minute_accuracy: 0.5404 - real_time_error: 0.7901
Epoch 00051: saving model to D:/study/models4/model-adam-51.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 12.5970 - hour_accuracy: 0.9890 - minute_accuracy: 0.5406 - real_time_error: 0.7893 - val_loss: 126.2980 - val_hour_accuracy: 0.7822 - val_minute_accuracy: 0.2147 - val_real_time_error: 27.3689
Epoch 52/100
14376/14400 [============================>.] - ETA: 0s - loss: 11.3263 - hour_accuracy: 0.9919 - minute_accuracy: 0.5876 - real_time_error: 0.6381
Epoch 00052: saving model to D:/study/models4/model-adam-52.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 11.3266 - hour_accuracy: 0.9918 - minute_accuracy: 0.5878 - real_time_error: 0.6377 - val_loss: 296.5351 - val_hour_accuracy: 0.4756 - val_minute_accuracy: 0.0822 - val_real_time_error: 47.2208
Epoch 53/100
14376/14400 [============================>.] - ETA: 0s - loss: 19.5555 - hour_accuracy: 0.9601 - minute_accuracy: 0.4838 - real_time_error: 2.9172
Epoch 00053: saving model to D:/study/models4/model-adam-53.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 19.5892 - hour_accuracy: 0.9600 - minute_accuracy: 0.4837 - real_time_error: 2.9187 - val_loss: 724.3835 - val_hour_accuracy: 0.2192 - val_minute_accuracy: 0.0528 - val_real_time_error: 76.4881
Epoch 54/100
14376/14400 [============================>.] - ETA: 0s - loss: 16.3879 - hour_accuracy: 0.9756 - minute_accuracy: 0.4500 - real_time_error: 1.7496
Epoch 00054: saving model to D:/study/models4/model-adam-54.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 16.3844 - hour_accuracy: 0.9756 - minute_accuracy: 0.4499 - real_time_error: 1.7480 - val_loss: 14.0991 - val_hour_accuracy: 0.9794 - val_minute_accuracy: 0.4350 - val_real_time_error: 0.9089
Epoch 55/100
14376/14400 [============================>.] - ETA: 0s - loss: 11.9189 - hour_accuracy: 0.9951 - minute_accuracy: 0.5678 - real_time_error: 0.5612
Epoch 00055: saving model to D:/study/models4/model-adam-55.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 11.9181 - hour_accuracy: 0.9951 - minute_accuracy: 0.5677 - real_time_error: 0.5610 - val_loss: 12.0704 - val_hour_accuracy: 0.9881 - val_minute_accuracy: 0.5128 - val_real_time_error: 0.6917
Epoch 56/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.8971 - hour_accuracy: 0.9957 - minute_accuracy: 0.6169 - real_time_error: 0.4754
Epoch 00056: saving model to D:/study/models4/model-adam-56.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 10.8952 - hour_accuracy: 0.9957 - minute_accuracy: 0.6173 - real_time_error: 0.4749 - val_loss: 12.3336 - val_hour_accuracy: 0.9858 - val_minute_accuracy: 0.5642 - val_real_time_error: 0.7653
Epoch 57/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.2521 - hour_accuracy: 0.9959 - minute_accuracy: 0.6354 - real_time_error: 0.4557
Epoch 00057: saving model to D:/study/models4/model-adam-57.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 10.2520 - hour_accuracy: 0.9959 - minute_accuracy: 0.6351 - real_time_error: 0.4559 - val_loss: 111.1459 - val_hour_accuracy: 0.6303 - val_minute_accuracy: 0.1817 - val_real_time_error: 24.6953
Epoch 58/100
14376/14400 [============================>.] - ETA: 0s - loss: 18.5787 - hour_accuracy: 0.9656 - minute_accuracy: 0.4854 - real_time_error: 2.6331
Epoch 00058: saving model to D:/study/models4/model-adam-58.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 18.5682 - hour_accuracy: 0.9656 - minute_accuracy: 0.4853 - real_time_error: 2.6299 - val_loss: 20.5007 - val_hour_accuracy: 0.9275 - val_minute_accuracy: 0.3628 - val_real_time_error: 4.2911
Epoch 59/100
14376/14400 [============================>.] - ETA: 0s - loss: 11.2690 - hour_accuracy: 0.9945 - minute_accuracy: 0.5884 - real_time_error: 0.5394
Epoch 00059: saving model to D:/study/models4/model-adam-59.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 11.2679 - hour_accuracy: 0.9945 - minute_accuracy: 0.5886 - real_time_error: 0.5390 - val_loss: 21.8149 - val_hour_accuracy: 0.9353 - val_minute_accuracy: 0.3400 - val_real_time_error: 4.3658
Epoch 60/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.2123 - hour_accuracy: 0.9951 - minute_accuracy: 0.6487 - real_time_error: 0.4465
Epoch 00060: saving model to D:/study/models4/model-adam-60.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 10.2121 - hour_accuracy: 0.9951 - minute_accuracy: 0.6484 - real_time_error: 0.4466 - val_loss: 12.9584 - val_hour_accuracy: 0.9661 - val_minute_accuracy: 0.5142 - val_real_time_error: 1.8553
Epoch 61/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.7269 - hour_accuracy: 0.9898 - minute_accuracy: 0.6285 - real_time_error: 0.7418
Epoch 00061: saving model to D:/study/models4/model-adam-61.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 10.7258 - hour_accuracy: 0.9899 - minute_accuracy: 0.6283 - real_time_error: 0.7415 - val_loss: 70.4665 - val_hour_accuracy: 0.7633 - val_minute_accuracy: 0.1883 - val_real_time_error: 15.3689
Epoch 62/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.2874 - hour_accuracy: 0.9734 - minute_accuracy: 0.5307 - real_time_error: 1.9679
Epoch 00062: saving model to D:/study/models4/model-adam-62.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 14.2804 - hour_accuracy: 0.9735 - minute_accuracy: 0.5309 - real_time_error: 1.9651 - val_loss: 22.5829 - val_hour_accuracy: 0.9283 - val_minute_accuracy: 0.2894 - val_real_time_error: 3.8539
Epoch 63/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.1632 - hour_accuracy: 0.9953 - minute_accuracy: 0.6491 - real_time_error: 0.4709
Epoch 00063: saving model to D:/study/models4/model-adam-63.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 10.1607 - hour_accuracy: 0.9953 - minute_accuracy: 0.6495 - real_time_error: 0.4703 - val_loss: 13.4061 - val_hour_accuracy: 0.9686 - val_minute_accuracy: 0.5639 - val_real_time_error: 1.7475
Epoch 64/100
14376/14400 [============================>.] - ETA: 0s - loss: 9.0018 - hour_accuracy: 0.9971 - minute_accuracy: 0.7113 - real_time_error: 0.3525
Epoch 00064: saving model to D:/study/models4/model-adam-64.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 9.0003 - hour_accuracy: 0.9972 - minute_accuracy: 0.7115 - real_time_error: 0.3522 - val_loss: 1310.7424 - val_hour_accuracy: 0.4936 - val_minute_accuracy: 0.1933 - val_real_time_error: 91.3647
Epoch 65/100
14376/14400 [============================>.] - ETA: 0s - loss: 20.2096 - hour_accuracy: 0.9558 - minute_accuracy: 0.5508 - real_time_error: 3.4404
Epoch 00065: saving model to D:/study/models4/model-adam-65.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 20.2086 - hour_accuracy: 0.9558 - minute_accuracy: 0.5506 - real_time_error: 3.4402 - val_loss: 1038.3417 - val_hour_accuracy: 0.2364 - val_minute_accuracy: 0.0453 - val_real_time_error: 93.0386
Epoch 66/100
14376/14400 [============================>.] - ETA: 0s - loss: 12.3517 - hour_accuracy: 0.9884 - minute_accuracy: 0.5520 - real_time_error: 0.9077
Epoch 00066: saving model to D:/study/models4/model-adam-66.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 12.3479 - hour_accuracy: 0.9884 - minute_accuracy: 0.5522 - real_time_error: 0.9068 - val_loss: 12.4205 - val_hour_accuracy: 0.9808 - val_minute_accuracy: 0.4686 - val_real_time_error: 0.8167
Epoch 67/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.0952 - hour_accuracy: 0.9960 - minute_accuracy: 0.6645 - real_time_error: 0.4343
Epoch 00067: saving model to D:/study/models4/model-adam-67.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 10.0950 - hour_accuracy: 0.9960 - minute_accuracy: 0.6645 - real_time_error: 0.4344 - val_loss: 10.0428 - val_hour_accuracy: 0.9944 - val_minute_accuracy: 0.6464 - val_real_time_error: 0.4083
Epoch 68/100
14376/14400 [============================>.] - ETA: 0s - loss: 9.1598 - hour_accuracy: 0.9983 - minute_accuracy: 0.7133 - real_time_error: 0.3340
Epoch 00068: saving model to D:/study/models4/model-adam-68.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 9.1592 - hour_accuracy: 0.9983 - minute_accuracy: 0.7132 - real_time_error: 0.3340 - val_loss: 9.4452 - val_hour_accuracy: 0.9925 - val_minute_accuracy: 0.6706 - val_real_time_error: 0.3775
Epoch 69/100
14376/14400 [============================>.] - ETA: 0s - loss: 8.4799 - hour_accuracy: 0.9987 - minute_accuracy: 0.7481 - real_time_error: 0.2976
Epoch 00069: saving model to D:/study/models4/model-adam-69.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 8.4796 - hour_accuracy: 0.9987 - minute_accuracy: 0.7479 - real_time_error: 0.2978 - val_loss: 12.3527 - val_hour_accuracy: 0.9758 - val_minute_accuracy: 0.5942 - val_real_time_error: 1.6739
Epoch 70/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.7105 - hour_accuracy: 0.9837 - minute_accuracy: 0.6664 - real_time_error: 1.0250
Epoch 00070: saving model to D:/study/models4/model-adam-70.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 10.7180 - hour_accuracy: 0.9837 - minute_accuracy: 0.6660 - real_time_error: 1.0249 - val_loss: 2865.6604 - val_hour_accuracy: 0.0428 - val_minute_accuracy: 0.0161 - val_real_time_error: 181.4669
Epoch 71/100
14376/14400 [============================>.] - ETA: 0s - loss: 19.6977 - hour_accuracy: 0.9613 - minute_accuracy: 0.4940 - real_time_error: 3.0366
Epoch 00071: saving model to D:/study/models4/model-adam-71.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 19.6818 - hour_accuracy: 0.9614 - minute_accuracy: 0.4944 - real_time_error: 3.0320 - val_loss: 14.7871 - val_hour_accuracy: 0.9775 - val_minute_accuracy: 0.4108 - val_real_time_error: 1.6925
Epoch 72/100
14376/14400 [============================>.] - ETA: 0s - loss: 11.9308 - hour_accuracy: 0.9859 - minute_accuracy: 0.6059 - real_time_error: 1.0480
Epoch 00072: saving model to D:/study/models4/model-adam-72.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 11.9277 - hour_accuracy: 0.9859 - minute_accuracy: 0.6058 - real_time_error: 1.0471 - val_loss: 12.0730 - val_hour_accuracy: 0.9864 - val_minute_accuracy: 0.5644 - val_real_time_error: 0.8128
Epoch 73/100
14376/14400 [============================>.] - ETA: 0s - loss: 9.5983 - hour_accuracy: 0.9960 - minute_accuracy: 0.6825 - real_time_error: 0.4636
Epoch 00073: saving model to D:/study/models4/model-adam-73.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 9.5970 - hour_accuracy: 0.9960 - minute_accuracy: 0.6828 - real_time_error: 0.4631 - val_loss: 10.5904 - val_hour_accuracy: 0.9875 - val_minute_accuracy: 0.5739 - val_real_time_error: 0.7325
Epoch 74/100
14376/14400 [============================>.] - ETA: 0s - loss: 8.7455 - hour_accuracy: 0.9968 - minute_accuracy: 0.7251 - real_time_error: 0.3731
Epoch 00074: saving model to D:/study/models4/model-adam-74.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 8.7445 - hour_accuracy: 0.9968 - minute_accuracy: 0.7253 - real_time_error: 0.3728 - val_loss: 8.9962 - val_hour_accuracy: 0.9925 - val_minute_accuracy: 0.6994 - val_real_time_error: 0.3614
Epoch 75/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.9330 - hour_accuracy: 0.9986 - minute_accuracy: 0.7700 - real_time_error: 0.2636
Epoch 00075: saving model to D:/study/models4/model-adam-75.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 7.9327 - hour_accuracy: 0.9986 - minute_accuracy: 0.7700 - real_time_error: 0.2635 - val_loss: 8.8313 - val_hour_accuracy: 0.9861 - val_minute_accuracy: 0.7189 - val_real_time_error: 0.3317
Epoch 76/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.8119 - hour_accuracy: 0.9834 - minute_accuracy: 0.6538 - real_time_error: 1.1490
Epoch 00076: saving model to D:/study/models4/model-adam-76.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 10.8084 - hour_accuracy: 0.9835 - minute_accuracy: 0.6538 - real_time_error: 1.1477 - val_loss: 119.8768 - val_hour_accuracy: 0.6514 - val_minute_accuracy: 0.1836 - val_real_time_error: 24.7325
Epoch 77/100
14376/14400 [============================>.] - ETA: 0s - loss: 8.4684 - hour_accuracy: 0.9958 - minute_accuracy: 0.7242 - real_time_error: 0.3744
Epoch 00077: saving model to D:/study/models4/model-adam-77.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 8.4667 - hour_accuracy: 0.9958 - minute_accuracy: 0.7244 - real_time_error: 0.3741 - val_loss: 83.2364 - val_hour_accuracy: 0.7333 - val_minute_accuracy: 0.2303 - val_real_time_error: 18.5306
Epoch 78/100
14376/14400 [============================>.] - ETA: 0s - loss: 14.4133 - hour_accuracy: 0.9737 - minute_accuracy: 0.5843 - real_time_error: 1.9775
Epoch 00078: saving model to D:/study/models4/model-adam-78.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 14.4066 - hour_accuracy: 0.9737 - minute_accuracy: 0.5845 - real_time_error: 1.9789 - val_loss: 16.6137 - val_hour_accuracy: 0.9700 - val_minute_accuracy: 0.5111 - val_real_time_error: 2.0456
Epoch 79/100
14376/14400 [============================>.] - ETA: 0s - loss: 9.1458 - hour_accuracy: 0.9942 - minute_accuracy: 0.6973 - real_time_error: 0.4846
Epoch 00079: saving model to D:/study/models4/model-adam-79.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 9.1459 - hour_accuracy: 0.9942 - minute_accuracy: 0.6972 - real_time_error: 0.4846 - val_loss: 15.0595 - val_hour_accuracy: 0.9542 - val_minute_accuracy: 0.4700 - val_real_time_error: 2.4017
Epoch 80/100
14376/14400 [============================>.] - ETA: 0s - loss: 8.0341 - hour_accuracy: 0.9974 - minute_accuracy: 0.7507 - real_time_error: 0.3015
Epoch 00080: saving model to D:/study/models4/model-adam-80.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 8.0331 - hour_accuracy: 0.9974 - minute_accuracy: 0.7508 - real_time_error: 0.3015 - val_loss: 20.6337 - val_hour_accuracy: 0.9364 - val_minute_accuracy: 0.4039 - val_real_time_error: 3.5864
Epoch 81/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.4237 - hour_accuracy: 0.9988 - minute_accuracy: 0.7901 - real_time_error: 0.2631
Epoch 00081: saving model to D:/study/models4/model-adam-81.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 7.4238 - hour_accuracy: 0.9988 - minute_accuracy: 0.7902 - real_time_error: 0.2629 - val_loss: 21.4868 - val_hour_accuracy: 0.9414 - val_minute_accuracy: 0.5969 - val_real_time_error: 8.0119
Epoch 82/100
14376/14400 [============================>.] - ETA: 0s - loss: 9.5792 - hour_accuracy: 0.9865 - minute_accuracy: 0.6795 - real_time_error: 0.9292
Epoch 00082: saving model to D:/study/models4/model-adam-82.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 9.5757 - hour_accuracy: 0.9865 - minute_accuracy: 0.6797 - real_time_error: 0.9280 - val_loss: 9.4428 - val_hour_accuracy: 0.9850 - val_minute_accuracy: 0.6314 - val_real_time_error: 0.6300
Epoch 83/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.6973 - hour_accuracy: 0.9958 - minute_accuracy: 0.7549 - real_time_error: 0.3963
Epoch 00083: saving model to D:/study/models4/model-adam-83.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 7.7016 - hour_accuracy: 0.9957 - minute_accuracy: 0.7544 - real_time_error: 0.3965 - val_loss: 460.2137 - val_hour_accuracy: 0.3603 - val_minute_accuracy: 0.0950 - val_real_time_error: 54.6478
Epoch 84/100
14376/14400 [============================>.] - ETA: 0s - loss: 10.4124 - hour_accuracy: 0.9819 - minute_accuracy: 0.6854 - real_time_error: 1.2762
Epoch 00084: saving model to D:/study/models4/model-adam-84.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 10.4124 - hour_accuracy: 0.9819 - minute_accuracy: 0.6853 - real_time_error: 1.2750 - val_loss: 406.9429 - val_hour_accuracy: 0.5522 - val_minute_accuracy: 0.1836 - val_real_time_error: 42.5719
Epoch 85/100
14376/14400 [============================>.] - ETA: 0s - loss: 12.6558 - hour_accuracy: 0.9802 - minute_accuracy: 0.6269 - real_time_error: 1.5624
Epoch 00085: saving model to D:/study/models4/model-adam-85.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 12.6496 - hour_accuracy: 0.9802 - minute_accuracy: 0.6269 - real_time_error: 1.5606 - val_loss: 10.9729 - val_hour_accuracy: 0.9817 - val_minute_accuracy: 0.5764 - val_real_time_error: 1.0794
Epoch 86/100
14376/14400 [============================>.] - ETA: 0s - loss: 8.2540 - hour_accuracy: 0.9965 - minute_accuracy: 0.7305 - real_time_error: 0.3750
Epoch 00086: saving model to D:/study/models4/model-adam-86.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 8.2530 - hour_accuracy: 0.9965 - minute_accuracy: 0.7306 - real_time_error: 0.3747 - val_loss: 8.1882 - val_hour_accuracy: 0.9936 - val_minute_accuracy: 0.7292 - val_real_time_error: 0.3275
Epoch 87/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.4420 - hour_accuracy: 0.9977 - minute_accuracy: 0.7783 - real_time_error: 0.3098
Epoch 00087: saving model to D:/study/models4/model-adam-87.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 7.4495 - hour_accuracy: 0.9976 - minute_accuracy: 0.7781 - real_time_error: 0.3100 - val_loss: 7.8087 - val_hour_accuracy: 0.9922 - val_minute_accuracy: 0.7464 - val_real_time_error: 0.3450
Epoch 88/100
14376/14400 [============================>.] - ETA: 0s - loss: 6.7582 - hour_accuracy: 0.9993 - minute_accuracy: 0.8152 - real_time_error: 0.2139
Epoch 00088: saving model to D:/study/models4/model-adam-88.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 6.7583 - hour_accuracy: 0.9993 - minute_accuracy: 0.8151 - real_time_error: 0.2140 - val_loss: 7.6505 - val_hour_accuracy: 0.9917 - val_minute_accuracy: 0.7283 - val_real_time_error: 0.4575
Epoch 89/100
14376/14400 [============================>.] - ETA: 0s - loss: 8.0517 - hour_accuracy: 0.9913 - minute_accuracy: 0.7423 - real_time_error: 0.6288
Epoch 00089: saving model to D:/study/models4/model-adam-89.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 8.0490 - hour_accuracy: 0.9913 - minute_accuracy: 0.7426 - real_time_error: 0.6278 - val_loss: 19.0694 - val_hour_accuracy: 0.9344 - val_minute_accuracy: 0.5503 - val_real_time_error: 2.8853
Epoch 90/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.4658 - hour_accuracy: 0.9933 - minute_accuracy: 0.7607 - real_time_error: 0.4880
Epoch 00090: saving model to D:/study/models4/model-adam-90.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 7.4636 - hour_accuracy: 0.9933 - minute_accuracy: 0.7610 - real_time_error: 0.4873 - val_loss: 13.4912 - val_hour_accuracy: 0.9644 - val_minute_accuracy: 0.6089 - val_real_time_error: 2.1033
Epoch 91/100
14376/14400 [============================>.] - ETA: 0s - loss: 11.2127 - hour_accuracy: 0.9827 - minute_accuracy: 0.6360 - real_time_error: 1.4556
Epoch 00091: saving model to D:/study/models4/model-adam-91.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 11.2079 - hour_accuracy: 0.9827 - minute_accuracy: 0.6362 - real_time_error: 1.4577 - val_loss: 9.4052 - val_hour_accuracy: 0.9881 - val_minute_accuracy: 0.5561 - val_real_time_error: 0.7039
Epoch 92/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.2206 - hour_accuracy: 0.9975 - minute_accuracy: 0.7735 - real_time_error: 0.2951
Epoch 00092: saving model to D:/study/models4/model-adam-92.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 7.2207 - hour_accuracy: 0.9975 - minute_accuracy: 0.7733 - real_time_error: 0.2952 - val_loss: 7.9326 - val_hour_accuracy: 0.9911 - val_minute_accuracy: 0.7506 - val_real_time_error: 0.4086
Epoch 93/100
14376/14400 [============================>.] - ETA: 0s - loss: 6.6553 - hour_accuracy: 0.9987 - minute_accuracy: 0.8029 - real_time_error: 0.2426
Epoch 00093: saving model to D:/study/models4/model-adam-93.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 6.6554 - hour_accuracy: 0.9987 - minute_accuracy: 0.8028 - real_time_error: 0.2426 - val_loss: 7.5212 - val_hour_accuracy: 0.9919 - val_minute_accuracy: 0.7150 - val_real_time_error: 0.4214
Epoch 94/100
14376/14400 [============================>.] - ETA: 0s - loss: 6.2221 - hour_accuracy: 0.9987 - minute_accuracy: 0.8246 - real_time_error: 0.2266
Epoch 00094: saving model to D:/study/models4/model-adam-94.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 6.2228 - hour_accuracy: 0.9987 - minute_accuracy: 0.8246 - real_time_error: 0.2266 - val_loss: 11.4200 - val_hour_accuracy: 0.9619 - val_minute_accuracy: 0.7161 - val_real_time_error: 2.0094
Epoch 95/100
14376/14400 [============================>.] - ETA: 0s - loss: 13.8321 - hour_accuracy: 0.9696 - minute_accuracy: 0.6596 - real_time_error: 2.3951
Epoch 00095: saving model to D:/study/models4/model-adam-95.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 13.8247 - hour_accuracy: 0.9697 - minute_accuracy: 0.6597 - real_time_error: 2.3919 - val_loss: 1127.3090 - val_hour_accuracy: 0.1964 - val_minute_accuracy: 0.0378 - val_real_time_error: 89.2167
Epoch 96/100
14376/14400 [============================>.] - ETA: 0s - loss: 9.1712 - hour_accuracy: 0.9903 - minute_accuracy: 0.6863 - real_time_error: 0.7957
Epoch 00096: saving model to D:/study/models4/model-adam-96.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 9.1682 - hour_accuracy: 0.9903 - minute_accuracy: 0.6862 - real_time_error: 0.7950 - val_loss: 8.5553 - val_hour_accuracy: 0.9908 - val_minute_accuracy: 0.6508 - val_real_time_error: 0.4289
Epoch 97/100
14376/14400 [============================>.] - ETA: 0s - loss: 7.1655 - hour_accuracy: 0.9985 - minute_accuracy: 0.7839 - real_time_error: 0.2533
Epoch 00097: saving model to D:/study/models4/model-adam-97.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 7.1651 - hour_accuracy: 0.9985 - minute_accuracy: 0.7839 - real_time_error: 0.2533 - val_loss: 7.4942 - val_hour_accuracy: 0.9956 - val_minute_accuracy: 0.7503 - val_real_time_error: 0.2944
Epoch 98/100
14376/14400 [============================>.] - ETA: 0s - loss: 6.5856 - hour_accuracy: 0.9995 - minute_accuracy: 0.8148 - real_time_error: 0.2177
Epoch 00098: saving model to D:/study/models4/model-adam-98.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 6.5847 - hour_accuracy: 0.9995 - minute_accuracy: 0.8149 - real_time_error: 0.2174 - val_loss: 7.1742 - val_hour_accuracy: 0.9944 - val_minute_accuracy: 0.7647 - val_real_time_error: 0.3125
Epoch 99/100
14376/14400 [============================>.] - ETA: 0s - loss: 6.2703 - hour_accuracy: 0.9987 - minute_accuracy: 0.8260 - real_time_error: 0.2104
Epoch 00099: saving model to D:/study/models4/model-adam-99.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 6.2695 - hour_accuracy: 0.9987 - minute_accuracy: 0.8261 - real_time_error: 0.2102 - val_loss: 7.0585 - val_hour_accuracy: 0.9925 - val_minute_accuracy: 0.7703 - val_real_time_error: 0.3344
Epoch 100/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.9095 - hour_accuracy: 0.9991 - minute_accuracy: 0.8452 - real_time_error: 0.1814
Epoch 00100: saving model to D:/study/models4/model-adam-100.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.9089 - hour_accuracy: 0.9991 - minute_accuracy: 0.8453 - real_time_error: 0.1813 - val_loss: 85.9410 - val_hour_accuracy: 0.7197 - val_minute_accuracy: 0.3156 - val_real_time_error: 18.2867
Train on 14400 samples, validate on 3600 samples
Epoch 1/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.6834 - hour_accuracy: 0.9991 - minute_accuracy: 0.8699 - real_time_error: 0.1546
Epoch 00001: saving model to D:/study/models4/model-sgd-01.hdf5
14400/14400 [==============================] - 118s 8ms/sample - loss: 5.6829 - hour_accuracy: 0.9991 - minute_accuracy: 0.8699 - real_time_error: 0.1545 - val_loss: 6.3368 - val_hour_accuracy: 0.9953 - val_minute_accuracy: 0.8242 - val_real_time_error: 0.2047
Epoch 2/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.6426 - hour_accuracy: 0.9993 - minute_accuracy: 0.8753 - real_time_error: 0.1418
Epoch 00002: saving model to D:/study/models4/model-sgd-02.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.6422 - hour_accuracy: 0.9993 - minute_accuracy: 0.8754 - real_time_error: 0.1417 - val_loss: 6.3211 - val_hour_accuracy: 0.9958 - val_minute_accuracy: 0.8386 - val_real_time_error: 0.2075
Epoch 3/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.6051 - hour_accuracy: 0.9996 - minute_accuracy: 0.8806 - real_time_error: 0.1357
Epoch 00003: saving model to D:/study/models4/model-sgd-03.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.6064 - hour_accuracy: 0.9996 - minute_accuracy: 0.8804 - real_time_error: 0.1360 - val_loss: 6.4593 - val_hour_accuracy: 0.9947 - val_minute_accuracy: 0.8061 - val_real_time_error: 0.2600
Epoch 4/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5830 - hour_accuracy: 0.9997 - minute_accuracy: 0.8871 - real_time_error: 0.1295
Epoch 00004: saving model to D:/study/models4/model-sgd-04.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5835 - hour_accuracy: 0.9997 - minute_accuracy: 0.8870 - real_time_error: 0.1296 - val_loss: 6.2083 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8486 - val_real_time_error: 0.1944
Epoch 5/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5798 - hour_accuracy: 0.9998 - minute_accuracy: 0.8891 - real_time_error: 0.1260
Epoch 00005: saving model to D:/study/models4/model-sgd-05.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5801 - hour_accuracy: 0.9998 - minute_accuracy: 0.8889 - real_time_error: 0.1261 - val_loss: 6.2450 - val_hour_accuracy: 0.9958 - val_minute_accuracy: 0.8369 - val_real_time_error: 0.2078
Epoch 6/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5897 - hour_accuracy: 0.9995 - minute_accuracy: 0.8868 - real_time_error: 0.1354
Epoch 00006: saving model to D:/study/models4/model-sgd-06.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5894 - hour_accuracy: 0.9995 - minute_accuracy: 0.8868 - real_time_error: 0.1353 - val_loss: 6.1987 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8431 - val_real_time_error: 0.1833
Epoch 7/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5830 - hour_accuracy: 0.9994 - minute_accuracy: 0.8879 - real_time_error: 0.1321
Epoch 00007: saving model to D:/study/models4/model-sgd-07.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5830 - hour_accuracy: 0.9994 - minute_accuracy: 0.8879 - real_time_error: 0.1320 - val_loss: 6.1773 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8525 - val_real_time_error: 0.1736
Epoch 8/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5802 - hour_accuracy: 0.9995 - minute_accuracy: 0.8934 - real_time_error: 0.1280
Epoch 00008: saving model to D:/study/models4/model-sgd-08.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5809 - hour_accuracy: 0.9995 - minute_accuracy: 0.8933 - real_time_error: 0.1281 - val_loss: 6.1982 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8511 - val_real_time_error: 0.1750
Epoch 9/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5450 - hour_accuracy: 0.9999 - minute_accuracy: 0.8934 - real_time_error: 0.1135
Epoch 00009: saving model to D:/study/models4/model-sgd-09.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5452 - hour_accuracy: 0.9999 - minute_accuracy: 0.8933 - real_time_error: 0.1135 - val_loss: 6.1557 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8528 - val_real_time_error: 0.1733
Epoch 10/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5462 - hour_accuracy: 0.9995 - minute_accuracy: 0.8935 - real_time_error: 0.1263
Epoch 00010: saving model to D:/study/models4/model-sgd-10.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5481 - hour_accuracy: 0.9994 - minute_accuracy: 0.8935 - real_time_error: 0.1263 - val_loss: 6.1589 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8533 - val_real_time_error: 0.1733
Epoch 11/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5516 - hour_accuracy: 0.9998 - minute_accuracy: 0.8920 - real_time_error: 0.1279
Epoch 00011: saving model to D:/study/models4/model-sgd-11.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5528 - hour_accuracy: 0.9998 - minute_accuracy: 0.8917 - real_time_error: 0.1281 - val_loss: 6.1987 - val_hour_accuracy: 0.9956 - val_minute_accuracy: 0.8472 - val_real_time_error: 0.1956
Epoch 12/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5765 - hour_accuracy: 0.9999 - minute_accuracy: 0.8922 - real_time_error: 0.1406
Epoch 00012: saving model to D:/study/models4/model-sgd-12.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5770 - hour_accuracy: 0.9999 - minute_accuracy: 0.8921 - real_time_error: 0.1406 - val_loss: 6.1502 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8536 - val_real_time_error: 0.1878
Epoch 13/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5517 - hour_accuracy: 0.9997 - minute_accuracy: 0.8922 - real_time_error: 0.1237
Epoch 00013: saving model to D:/study/models4/model-sgd-13.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5514 - hour_accuracy: 0.9997 - minute_accuracy: 0.8922 - real_time_error: 0.1236 - val_loss: 6.1573 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8528 - val_real_time_error: 0.1894
Epoch 14/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5310 - hour_accuracy: 0.9996 - minute_accuracy: 0.8954 - real_time_error: 0.1146
Epoch 00014: saving model to D:/study/models4/model-sgd-14.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5308 - hour_accuracy: 0.9996 - minute_accuracy: 0.8953 - real_time_error: 0.1146 - val_loss: 6.1896 - val_hour_accuracy: 0.9953 - val_minute_accuracy: 0.8453 - val_real_time_error: 0.1986
Epoch 15/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5360 - hour_accuracy: 0.9999 - minute_accuracy: 0.8945 - real_time_error: 0.1141
Epoch 00015: saving model to D:/study/models4/model-sgd-15.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5355 - hour_accuracy: 0.9999 - minute_accuracy: 0.8947 - real_time_error: 0.1140 - val_loss: 6.1548 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1686
Epoch 16/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5344 - hour_accuracy: 0.9999 - minute_accuracy: 0.8935 - real_time_error: 0.1153
Epoch 00016: saving model to D:/study/models4/model-sgd-16.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5341 - hour_accuracy: 0.9999 - minute_accuracy: 0.8936 - real_time_error: 0.1152 - val_loss: 6.1442 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8539 - val_real_time_error: 0.1714
Epoch 17/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5309 - hour_accuracy: 0.9997 - minute_accuracy: 0.8932 - real_time_error: 0.1173
Epoch 00017: saving model to D:/study/models4/model-sgd-17.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5306 - hour_accuracy: 0.9997 - minute_accuracy: 0.8933 - real_time_error: 0.1172 - val_loss: 6.1483 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8522 - val_real_time_error: 0.1894
Epoch 18/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5136 - hour_accuracy: 0.9998 - minute_accuracy: 0.8982 - real_time_error: 0.1128
Epoch 00018: saving model to D:/study/models4/model-sgd-18.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5133 - hour_accuracy: 0.9998 - minute_accuracy: 0.8982 - real_time_error: 0.1128 - val_loss: 6.1397 - val_hour_accuracy: 0.9972 - val_minute_accuracy: 0.8558 - val_real_time_error: 0.1700
Epoch 19/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5367 - hour_accuracy: 0.9999 - minute_accuracy: 0.8948 - real_time_error: 0.1192
Epoch 00019: saving model to D:/study/models4/model-sgd-19.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5362 - hour_accuracy: 0.9999 - minute_accuracy: 0.8950 - real_time_error: 0.1190 - val_loss: 6.1367 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8544 - val_real_time_error: 0.1708
Epoch 20/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5293 - hour_accuracy: 0.9998 - minute_accuracy: 0.8943 - real_time_error: 0.1171
Epoch 00020: saving model to D:/study/models4/model-sgd-20.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5293 - hour_accuracy: 0.9998 - minute_accuracy: 0.8940 - real_time_error: 0.1174 - val_loss: 6.1367 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1686
Epoch 21/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5114 - hour_accuracy: 0.9997 - minute_accuracy: 0.8989 - real_time_error: 0.1175
Epoch 00021: saving model to D:/study/models4/model-sgd-21.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5112 - hour_accuracy: 0.9997 - minute_accuracy: 0.8990 - real_time_error: 0.1174 - val_loss: 6.1368 - val_hour_accuracy: 0.9958 - val_minute_accuracy: 0.8550 - val_real_time_error: 0.1867
Epoch 22/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5038 - hour_accuracy: 0.9998 - minute_accuracy: 0.8996 - real_time_error: 0.1146
Epoch 00022: saving model to D:/study/models4/model-sgd-22.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5038 - hour_accuracy: 0.9998 - minute_accuracy: 0.8995 - real_time_error: 0.1147 - val_loss: 6.1286 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1514
Epoch 23/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5184 - hour_accuracy: 0.9999 - minute_accuracy: 0.8955 - real_time_error: 0.1193
Epoch 00023: saving model to D:/study/models4/model-sgd-23.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5179 - hour_accuracy: 0.9999 - minute_accuracy: 0.8957 - real_time_error: 0.1191 - val_loss: 6.1293 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8544 - val_real_time_error: 0.1703
Epoch 24/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5079 - hour_accuracy: 0.9999 - minute_accuracy: 0.9000 - real_time_error: 0.1060
Epoch 00024: saving model to D:/study/models4/model-sgd-24.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5075 - hour_accuracy: 0.9999 - minute_accuracy: 0.9001 - real_time_error: 0.1059 - val_loss: 6.1266 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8556 - val_real_time_error: 0.1694
Epoch 25/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5112 - hour_accuracy: 0.9997 - minute_accuracy: 0.9003 - real_time_error: 0.1187
Epoch 00025: saving model to D:/study/models4/model-sgd-25.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5106 - hour_accuracy: 0.9997 - minute_accuracy: 0.9003 - real_time_error: 0.1185 - val_loss: 6.1259 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8556 - val_real_time_error: 0.1700
Epoch 26/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5296 - hour_accuracy: 0.9998 - minute_accuracy: 0.8939 - real_time_error: 0.1231
Epoch 00026: saving model to D:/study/models4/model-sgd-26.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5292 - hour_accuracy: 0.9998 - minute_accuracy: 0.8939 - real_time_error: 0.1230 - val_loss: 6.1481 - val_hour_accuracy: 0.9958 - val_minute_accuracy: 0.8539 - val_real_time_error: 0.1711
Epoch 27/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5183 - hour_accuracy: 0.9998 - minute_accuracy: 0.8972 - real_time_error: 0.1135
Epoch 00027: saving model to D:/study/models4/model-sgd-27.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5181 - hour_accuracy: 0.9998 - minute_accuracy: 0.8972 - real_time_error: 0.1134 - val_loss: 6.1213 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8592 - val_real_time_error: 0.1494
Epoch 28/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5220 - hour_accuracy: 0.9998 - minute_accuracy: 0.8948 - real_time_error: 0.1230
Epoch 00028: saving model to D:/study/models4/model-sgd-28.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5217 - hour_accuracy: 0.9998 - minute_accuracy: 0.8949 - real_time_error: 0.1228 - val_loss: 6.1212 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1675
Epoch 29/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5040 - hour_accuracy: 0.9999 - minute_accuracy: 0.8993 - real_time_error: 0.1114
Epoch 00029: saving model to D:/study/models4/model-sgd-29.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5038 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1113 - val_loss: 6.1221 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8564 - val_real_time_error: 0.1856
Epoch 30/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5090 - hour_accuracy: 0.9999 - minute_accuracy: 0.8980 - real_time_error: 0.1135
Epoch 00030: saving model to D:/study/models4/model-sgd-30.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5086 - hour_accuracy: 0.9999 - minute_accuracy: 0.8981 - real_time_error: 0.1134 - val_loss: 6.1117 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1519
Epoch 31/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5017 - hour_accuracy: 0.9999 - minute_accuracy: 0.9008 - real_time_error: 0.1093
Epoch 00031: saving model to D:/study/models4/model-sgd-31.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5016 - hour_accuracy: 0.9999 - minute_accuracy: 0.9008 - real_time_error: 0.1093 - val_loss: 6.1146 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8561 - val_real_time_error: 0.1533
Epoch 32/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5055 - hour_accuracy: 0.9999 - minute_accuracy: 0.8973 - real_time_error: 0.1099
Epoch 00032: saving model to D:/study/models4/model-sgd-32.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5051 - hour_accuracy: 0.9999 - minute_accuracy: 0.8974 - real_time_error: 0.1098 - val_loss: 6.1135 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1506
Epoch 33/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5159 - hour_accuracy: 0.9999 - minute_accuracy: 0.8980 - real_time_error: 0.1094
Epoch 00033: saving model to D:/study/models4/model-sgd-33.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5157 - hour_accuracy: 0.9999 - minute_accuracy: 0.8979 - real_time_error: 0.1094 - val_loss: 6.1179 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8575 - val_real_time_error: 0.1678
Epoch 34/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4971 - hour_accuracy: 1.0000 - minute_accuracy: 0.8966 - real_time_error: 0.1096
Epoch 00034: saving model to D:/study/models4/model-sgd-34.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4970 - hour_accuracy: 1.0000 - minute_accuracy: 0.8967 - real_time_error: 0.1096 - val_loss: 6.1160 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1508
Epoch 35/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5097 - hour_accuracy: 0.9997 - minute_accuracy: 0.8976 - real_time_error: 0.1169
Epoch 00035: saving model to D:/study/models4/model-sgd-35.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5098 - hour_accuracy: 0.9997 - minute_accuracy: 0.8976 - real_time_error: 0.1169 - val_loss: 6.1093 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8592 - val_real_time_error: 0.1500
Epoch 36/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5045 - hour_accuracy: 0.9999 - minute_accuracy: 0.9013 - real_time_error: 0.1089
Epoch 00036: saving model to D:/study/models4/model-sgd-36.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5046 - hour_accuracy: 0.9999 - minute_accuracy: 0.9012 - real_time_error: 0.1090 - val_loss: 6.1088 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8575 - val_real_time_error: 0.1517
Epoch 37/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5161 - hour_accuracy: 0.9997 - minute_accuracy: 0.8945 - real_time_error: 0.1155
Epoch 00037: saving model to D:/study/models4/model-sgd-37.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5156 - hour_accuracy: 0.9997 - minute_accuracy: 0.8946 - real_time_error: 0.1153 - val_loss: 6.1114 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8597 - val_real_time_error: 0.1489
Epoch 38/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4920 - hour_accuracy: 0.9999 - minute_accuracy: 0.9012 - real_time_error: 0.1055
Epoch 00038: saving model to D:/study/models4/model-sgd-38.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4927 - hour_accuracy: 0.9999 - minute_accuracy: 0.9010 - real_time_error: 0.1060 - val_loss: 6.1054 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8603 - val_real_time_error: 0.1486
Epoch 39/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5220 - hour_accuracy: 0.9996 - minute_accuracy: 0.8997 - real_time_error: 0.1108
Epoch 00039: saving model to D:/study/models4/model-sgd-39.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5218 - hour_accuracy: 0.9996 - minute_accuracy: 0.8997 - real_time_error: 0.1108 - val_loss: 6.1131 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1511
Epoch 40/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5175 - hour_accuracy: 0.9999 - minute_accuracy: 0.8996 - real_time_error: 0.1327
Epoch 00040: saving model to D:/study/models4/model-sgd-40.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5179 - hour_accuracy: 0.9999 - minute_accuracy: 0.8997 - real_time_error: 0.1326 - val_loss: 6.1131 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8583 - val_real_time_error: 0.1836
Epoch 41/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4906 - hour_accuracy: 0.9999 - minute_accuracy: 0.9019 - real_time_error: 0.1089
Epoch 00041: saving model to D:/study/models4/model-sgd-41.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4908 - hour_accuracy: 0.9999 - minute_accuracy: 0.9018 - real_time_error: 0.1090 - val_loss: 6.1084 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1519
Epoch 42/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4818 - hour_accuracy: 0.9999 - minute_accuracy: 0.9030 - real_time_error: 0.1077
Epoch 00042: saving model to D:/study/models4/model-sgd-42.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4818 - hour_accuracy: 0.9999 - minute_accuracy: 0.9030 - real_time_error: 0.1076 - val_loss: 6.1101 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8569 - val_real_time_error: 0.1519
Epoch 43/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5022 - hour_accuracy: 0.9999 - minute_accuracy: 0.8976 - real_time_error: 0.1100
Epoch 00043: saving model to D:/study/models4/model-sgd-43.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5023 - hour_accuracy: 0.9999 - minute_accuracy: 0.8975 - real_time_error: 0.1101 - val_loss: 6.1029 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8561 - val_real_time_error: 0.1531
Epoch 44/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4997 - hour_accuracy: 0.9998 - minute_accuracy: 0.9003 - real_time_error: 0.1106
Epoch 00044: saving model to D:/study/models4/model-sgd-44.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5001 - hour_accuracy: 0.9998 - minute_accuracy: 0.9003 - real_time_error: 0.1106 - val_loss: 6.1058 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8594 - val_real_time_error: 0.1497
Epoch 45/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5042 - hour_accuracy: 0.9996 - minute_accuracy: 0.9015 - real_time_error: 0.1098
Epoch 00045: saving model to D:/study/models4/model-sgd-45.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5046 - hour_accuracy: 0.9996 - minute_accuracy: 0.9015 - real_time_error: 0.1098 - val_loss: 6.1121 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1678
Epoch 46/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5033 - hour_accuracy: 0.9998 - minute_accuracy: 0.9023 - real_time_error: 0.1128
Epoch 00046: saving model to D:/study/models4/model-sgd-46.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5031 - hour_accuracy: 0.9998 - minute_accuracy: 0.9022 - real_time_error: 0.1128 - val_loss: 6.1061 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8575 - val_real_time_error: 0.1675
Epoch 47/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5130 - hour_accuracy: 0.9997 - minute_accuracy: 0.8991 - real_time_error: 0.1110
Epoch 00047: saving model to D:/study/models4/model-sgd-47.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5125 - hour_accuracy: 0.9997 - minute_accuracy: 0.8992 - real_time_error: 0.1109 - val_loss: 6.1102 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1500
Epoch 48/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5051 - hour_accuracy: 0.9998 - minute_accuracy: 0.8982 - real_time_error: 0.1151
Epoch 00048: saving model to D:/study/models4/model-sgd-48.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5046 - hour_accuracy: 0.9998 - minute_accuracy: 0.8982 - real_time_error: 0.1151 - val_loss: 6.1107 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1519
Epoch 49/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4809 - hour_accuracy: 0.9999 - minute_accuracy: 0.9033 - real_time_error: 0.1027
Epoch 00049: saving model to D:/study/models4/model-sgd-49.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4808 - hour_accuracy: 0.9999 - minute_accuracy: 0.9033 - real_time_error: 0.1027 - val_loss: 6.1048 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1503
Epoch 50/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5045 - hour_accuracy: 0.9998 - minute_accuracy: 0.8982 - real_time_error: 0.1126
Epoch 00050: saving model to D:/study/models4/model-sgd-50.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5042 - hour_accuracy: 0.9998 - minute_accuracy: 0.8984 - real_time_error: 0.1124 - val_loss: 6.1057 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8614 - val_real_time_error: 0.1478
Epoch 51/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5055 - hour_accuracy: 0.9999 - minute_accuracy: 0.8973 - real_time_error: 0.1137
Epoch 00051: saving model to D:/study/models4/model-sgd-51.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5049 - hour_accuracy: 0.9999 - minute_accuracy: 0.8974 - real_time_error: 0.1135 - val_loss: 6.1051 - val_hour_accuracy: 0.9961 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1672
Epoch 52/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4979 - hour_accuracy: 0.9998 - minute_accuracy: 0.9010 - real_time_error: 0.1046
Epoch 00052: saving model to D:/study/models4/model-sgd-52.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.4977 - hour_accuracy: 0.9998 - minute_accuracy: 0.9011 - real_time_error: 0.1046 - val_loss: 6.1070 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8583 - val_real_time_error: 0.1506
Epoch 53/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4972 - hour_accuracy: 0.9997 - minute_accuracy: 0.8993 - real_time_error: 0.1148
Epoch 00053: saving model to D:/study/models4/model-sgd-53.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4980 - hour_accuracy: 0.9997 - minute_accuracy: 0.8991 - real_time_error: 0.1151 - val_loss: 6.0987 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8594 - val_real_time_error: 0.1494
Epoch 54/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4945 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1100
Epoch 00054: saving model to D:/study/models4/model-sgd-54.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4947 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1100 - val_loss: 6.1080 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1675
Epoch 55/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5012 - hour_accuracy: 0.9996 - minute_accuracy: 0.9016 - real_time_error: 0.1123
Epoch 00055: saving model to D:/study/models4/model-sgd-55.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5012 - hour_accuracy: 0.9996 - minute_accuracy: 0.9015 - real_time_error: 0.1124 - val_loss: 6.1034 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8592 - val_real_time_error: 0.1503
Epoch 56/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4810 - hour_accuracy: 0.9999 - minute_accuracy: 0.9032 - real_time_error: 0.1028
Epoch 00056: saving model to D:/study/models4/model-sgd-56.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4808 - hour_accuracy: 0.9999 - minute_accuracy: 0.9033 - real_time_error: 0.1027 - val_loss: 6.1018 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8600 - val_real_time_error: 0.1486
Epoch 57/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4900 - hour_accuracy: 0.9998 - minute_accuracy: 0.8998 - real_time_error: 0.1061
Epoch 00057: saving model to D:/study/models4/model-sgd-57.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4900 - hour_accuracy: 0.9998 - minute_accuracy: 0.8997 - real_time_error: 0.1063 - val_loss: 6.1020 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1514
Epoch 58/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4849 - hour_accuracy: 0.9999 - minute_accuracy: 0.9026 - real_time_error: 0.1021
Epoch 00058: saving model to D:/study/models4/model-sgd-58.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.4847 - hour_accuracy: 0.9999 - minute_accuracy: 0.9026 - real_time_error: 0.1021 - val_loss: 6.1065 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1503
Epoch 59/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5009 - hour_accuracy: 0.9998 - minute_accuracy: 0.9007 - real_time_error: 0.1148
Epoch 00059: saving model to D:/study/models4/model-sgd-59.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5004 - hour_accuracy: 0.9998 - minute_accuracy: 0.9008 - real_time_error: 0.1147 - val_loss: 6.1013 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8603 - val_real_time_error: 0.1647
Epoch 60/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4921 - hour_accuracy: 0.9999 - minute_accuracy: 0.8957 - real_time_error: 0.1109
Epoch 00060: saving model to D:/study/models4/model-sgd-60.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4917 - hour_accuracy: 0.9999 - minute_accuracy: 0.8959 - real_time_error: 0.1108 - val_loss: 6.1038 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8589 - val_real_time_error: 0.1500
Epoch 61/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5102 - hour_accuracy: 0.9999 - minute_accuracy: 0.8970 - real_time_error: 0.1143
Epoch 00061: saving model to D:/study/models4/model-sgd-61.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5100 - hour_accuracy: 0.9999 - minute_accuracy: 0.8969 - real_time_error: 0.1143 - val_loss: 6.1030 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1672
Epoch 62/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4838 - hour_accuracy: 0.9999 - minute_accuracy: 0.9018 - real_time_error: 0.1071
Epoch 00062: saving model to D:/study/models4/model-sgd-62.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4834 - hour_accuracy: 0.9999 - minute_accuracy: 0.9018 - real_time_error: 0.1071 - val_loss: 6.1056 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8603 - val_real_time_error: 0.1489
Epoch 63/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4852 - hour_accuracy: 1.0000 - minute_accuracy: 0.9007 - real_time_error: 0.1075
Epoch 00063: saving model to D:/study/models4/model-sgd-63.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4851 - hour_accuracy: 1.0000 - minute_accuracy: 0.9006 - real_time_error: 0.1076 - val_loss: 6.0968 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1508
Epoch 64/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4890 - hour_accuracy: 0.9997 - minute_accuracy: 0.9009 - real_time_error: 0.1180
Epoch 00064: saving model to D:/study/models4/model-sgd-64.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4890 - hour_accuracy: 0.9997 - minute_accuracy: 0.9010 - real_time_error: 0.1179 - val_loss: 6.1021 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8589 - val_real_time_error: 0.1503
Epoch 65/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5058 - hour_accuracy: 0.9999 - minute_accuracy: 0.8979 - real_time_error: 0.1125
Epoch 00065: saving model to D:/study/models4/model-sgd-65.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5057 - hour_accuracy: 0.9999 - minute_accuracy: 0.8978 - real_time_error: 0.1126 - val_loss: 6.1051 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8597 - val_real_time_error: 0.1497
Epoch 66/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4911 - hour_accuracy: 0.9998 - minute_accuracy: 0.8995 - real_time_error: 0.1106
Epoch 00066: saving model to D:/study/models4/model-sgd-66.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4906 - hour_accuracy: 0.9998 - minute_accuracy: 0.8997 - real_time_error: 0.1104 - val_loss: 6.1011 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8583 - val_real_time_error: 0.1511
Epoch 67/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4840 - hour_accuracy: 0.9997 - minute_accuracy: 0.9041 - real_time_error: 0.1119
Epoch 00067: saving model to D:/study/models4/model-sgd-67.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4836 - hour_accuracy: 0.9997 - minute_accuracy: 0.9042 - real_time_error: 0.1119 - val_loss: 6.1023 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1503
Epoch 68/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4948 - hour_accuracy: 0.9997 - minute_accuracy: 0.8990 - real_time_error: 0.1162
Epoch 00068: saving model to D:/study/models4/model-sgd-68.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4956 - hour_accuracy: 0.9997 - minute_accuracy: 0.8989 - real_time_error: 0.1163 - val_loss: 6.1044 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1514
Epoch 69/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5100 - hour_accuracy: 0.9999 - minute_accuracy: 0.8961 - real_time_error: 0.1111
Epoch 00069: saving model to D:/study/models4/model-sgd-69.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5099 - hour_accuracy: 0.9999 - minute_accuracy: 0.8961 - real_time_error: 0.1110 - val_loss: 6.1039 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1508
Epoch 70/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5049 - hour_accuracy: 0.9997 - minute_accuracy: 0.9013 - real_time_error: 0.1139
Epoch 00070: saving model to D:/study/models4/model-sgd-70.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5051 - hour_accuracy: 0.9997 - minute_accuracy: 0.9013 - real_time_error: 0.1138 - val_loss: 6.1045 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1506
Epoch 71/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4929 - hour_accuracy: 0.9999 - minute_accuracy: 0.9028 - real_time_error: 0.1077
Epoch 00071: saving model to D:/study/models4/model-sgd-71.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4926 - hour_accuracy: 0.9999 - minute_accuracy: 0.9028 - real_time_error: 0.1076 - val_loss: 6.1010 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8594 - val_real_time_error: 0.1494
Epoch 72/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5039 - hour_accuracy: 0.9996 - minute_accuracy: 0.9007 - real_time_error: 0.1217
Epoch 00072: saving model to D:/study/models4/model-sgd-72.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5042 - hour_accuracy: 0.9996 - minute_accuracy: 0.9006 - real_time_error: 0.1218 - val_loss: 6.1027 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8561 - val_real_time_error: 0.1533
Epoch 73/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5098 - hour_accuracy: 0.9997 - minute_accuracy: 0.8961 - real_time_error: 0.1226
Epoch 00073: saving model to D:/study/models4/model-sgd-73.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5095 - hour_accuracy: 0.9997 - minute_accuracy: 0.8962 - real_time_error: 0.1224 - val_loss: 6.0970 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8589 - val_real_time_error: 0.1497
Epoch 74/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4899 - hour_accuracy: 0.9996 - minute_accuracy: 0.9024 - real_time_error: 0.1067
Epoch 00074: saving model to D:/study/models4/model-sgd-74.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4904 - hour_accuracy: 0.9996 - minute_accuracy: 0.9024 - real_time_error: 0.1067 - val_loss: 6.1020 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1519
Epoch 75/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4891 - hour_accuracy: 0.9998 - minute_accuracy: 0.9003 - real_time_error: 0.1151
Epoch 00075: saving model to D:/study/models4/model-sgd-75.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4892 - hour_accuracy: 0.9998 - minute_accuracy: 0.9001 - real_time_error: 0.1152 - val_loss: 6.1014 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8583 - val_real_time_error: 0.1514
Epoch 76/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4965 - hour_accuracy: 0.9999 - minute_accuracy: 0.9002 - real_time_error: 0.1060
Epoch 00076: saving model to D:/study/models4/model-sgd-76.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4963 - hour_accuracy: 0.9999 - minute_accuracy: 0.9002 - real_time_error: 0.1060 - val_loss: 6.1020 - val_hour_accuracy: 0.9972 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1506
Epoch 77/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4917 - hour_accuracy: 0.9997 - minute_accuracy: 0.9004 - real_time_error: 0.1053
Epoch 00077: saving model to D:/study/models4/model-sgd-77.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4916 - hour_accuracy: 0.9997 - minute_accuracy: 0.9004 - real_time_error: 0.1053 - val_loss: 6.1044 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1514
Epoch 78/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4971 - hour_accuracy: 0.9999 - minute_accuracy: 0.9016 - real_time_error: 0.1091
Epoch 00078: saving model to D:/study/models4/model-sgd-78.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4974 - hour_accuracy: 0.9999 - minute_accuracy: 0.9015 - real_time_error: 0.1092 - val_loss: 6.1057 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8569 - val_real_time_error: 0.1522
Epoch 79/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4932 - hour_accuracy: 0.9999 - minute_accuracy: 0.8995 - real_time_error: 0.1111
Epoch 00079: saving model to D:/study/models4/model-sgd-79.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4931 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1112 - val_loss: 6.1041 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1514
Epoch 80/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4926 - hour_accuracy: 0.9999 - minute_accuracy: 0.9003 - real_time_error: 0.1103
Epoch 00080: saving model to D:/study/models4/model-sgd-80.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.4925 - hour_accuracy: 0.9999 - minute_accuracy: 0.9003 - real_time_error: 0.1101 - val_loss: 6.0984 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8572 - val_real_time_error: 0.1519
Epoch 81/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4953 - hour_accuracy: 0.9999 - minute_accuracy: 0.8959 - real_time_error: 0.1145
Epoch 00081: saving model to D:/study/models4/model-sgd-81.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.4953 - hour_accuracy: 0.9999 - minute_accuracy: 0.8960 - real_time_error: 0.1144 - val_loss: 6.1020 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8589 - val_real_time_error: 0.1503
Epoch 82/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4976 - hour_accuracy: 0.9999 - minute_accuracy: 0.8975 - real_time_error: 0.1089
Epoch 00082: saving model to D:/study/models4/model-sgd-82.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.4974 - hour_accuracy: 0.9999 - minute_accuracy: 0.8975 - real_time_error: 0.1088 - val_loss: 6.1056 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1514
Epoch 83/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4826 - hour_accuracy: 1.0000 - minute_accuracy: 0.9009 - real_time_error: 0.1043
Epoch 00083: saving model to D:/study/models4/model-sgd-83.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4822 - hour_accuracy: 1.0000 - minute_accuracy: 0.9010 - real_time_error: 0.1042 - val_loss: 6.1045 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1511
Epoch 84/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5087 - hour_accuracy: 0.9998 - minute_accuracy: 0.8954 - real_time_error: 0.1235
Epoch 00084: saving model to D:/study/models4/model-sgd-84.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.5084 - hour_accuracy: 0.9998 - minute_accuracy: 0.8955 - real_time_error: 0.1234 - val_loss: 6.1006 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8592 - val_real_time_error: 0.1500
Epoch 85/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4753 - hour_accuracy: 0.9999 - minute_accuracy: 0.9016 - real_time_error: 0.1039
Epoch 00085: saving model to D:/study/models4/model-sgd-85.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4756 - hour_accuracy: 0.9999 - minute_accuracy: 0.9016 - real_time_error: 0.1039 - val_loss: 6.0987 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8575 - val_real_time_error: 0.1514
Epoch 86/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4963 - hour_accuracy: 0.9999 - minute_accuracy: 0.8960 - real_time_error: 0.1155
Epoch 00086: saving model to D:/study/models4/model-sgd-86.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4959 - hour_accuracy: 0.9999 - minute_accuracy: 0.8961 - real_time_error: 0.1154 - val_loss: 6.1027 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1506
Epoch 87/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4928 - hour_accuracy: 0.9998 - minute_accuracy: 0.9009 - real_time_error: 0.1093
Epoch 00087: saving model to D:/study/models4/model-sgd-87.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4935 - hour_accuracy: 0.9998 - minute_accuracy: 0.9007 - real_time_error: 0.1094 - val_loss: 6.1026 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8589 - val_real_time_error: 0.1503
Epoch 88/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4891 - hour_accuracy: 0.9999 - minute_accuracy: 0.9001 - real_time_error: 0.1050
Epoch 00088: saving model to D:/study/models4/model-sgd-88.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4892 - hour_accuracy: 0.9999 - minute_accuracy: 0.9001 - real_time_error: 0.1050 - val_loss: 6.1034 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8578 - val_real_time_error: 0.1511
Epoch 89/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5053 - hour_accuracy: 0.9999 - minute_accuracy: 0.8984 - real_time_error: 0.1075
Epoch 00089: saving model to D:/study/models4/model-sgd-89.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5052 - hour_accuracy: 0.9999 - minute_accuracy: 0.8984 - real_time_error: 0.1076 - val_loss: 6.0998 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8597 - val_real_time_error: 0.1494
Epoch 90/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4816 - hour_accuracy: 0.9999 - minute_accuracy: 0.9005 - real_time_error: 0.1055
Epoch 00090: saving model to D:/study/models4/model-sgd-90.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4818 - hour_accuracy: 0.9999 - minute_accuracy: 0.9004 - real_time_error: 0.1056 - val_loss: 6.1017 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1508
Epoch 91/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4801 - hour_accuracy: 0.9999 - minute_accuracy: 0.9046 - real_time_error: 0.1099
Epoch 00091: saving model to D:/study/models4/model-sgd-91.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4800 - hour_accuracy: 0.9999 - minute_accuracy: 0.9047 - real_time_error: 0.1100 - val_loss: 6.1018 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1514
Epoch 92/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4890 - hour_accuracy: 0.9997 - minute_accuracy: 0.9035 - real_time_error: 0.1099
Epoch 00092: saving model to D:/study/models4/model-sgd-92.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4893 - hour_accuracy: 0.9997 - minute_accuracy: 0.9033 - real_time_error: 0.1101 - val_loss: 6.0997 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8592 - val_real_time_error: 0.1497
Epoch 93/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4945 - hour_accuracy: 0.9999 - minute_accuracy: 0.8993 - real_time_error: 0.1104
Epoch 00093: saving model to D:/study/models4/model-sgd-93.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4942 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1103 - val_loss: 6.1004 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1503
Epoch 94/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4978 - hour_accuracy: 0.9996 - minute_accuracy: 0.9008 - real_time_error: 0.1207
Epoch 00094: saving model to D:/study/models4/model-sgd-94.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4977 - hour_accuracy: 0.9996 - minute_accuracy: 0.9008 - real_time_error: 0.1206 - val_loss: 6.0988 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8594 - val_real_time_error: 0.1497
Epoch 95/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4867 - hour_accuracy: 0.9999 - minute_accuracy: 0.9019 - real_time_error: 0.1089
Epoch 00095: saving model to D:/study/models4/model-sgd-95.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4866 - hour_accuracy: 0.9999 - minute_accuracy: 0.9019 - real_time_error: 0.1089 - val_loss: 6.1028 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8583 - val_real_time_error: 0.1511
Epoch 96/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4942 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1087
Epoch 00096: saving model to D:/study/models4/model-sgd-96.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4941 - hour_accuracy: 0.9999 - minute_accuracy: 0.8994 - real_time_error: 0.1088 - val_loss: 6.1016 - val_hour_accuracy: 0.9964 - val_minute_accuracy: 0.8589 - val_real_time_error: 0.1506
Epoch 97/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4883 - hour_accuracy: 1.0000 - minute_accuracy: 0.9022 - real_time_error: 0.1053
Epoch 00097: saving model to D:/study/models4/model-sgd-97.hdf5
14400/14400 [==============================] - 106s 7ms/sample - loss: 5.4883 - hour_accuracy: 1.0000 - minute_accuracy: 0.9022 - real_time_error: 0.1053 - val_loss: 6.0982 - val_hour_accuracy: 0.9969 - val_minute_accuracy: 0.8586 - val_real_time_error: 0.1664
Epoch 98/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5052 - hour_accuracy: 0.9999 - minute_accuracy: 0.8975 - real_time_error: 0.1107
Epoch 00098: saving model to D:/study/models4/model-sgd-98.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5049 - hour_accuracy: 0.9999 - minute_accuracy: 0.8976 - real_time_error: 0.1106 - val_loss: 6.0979 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8600 - val_real_time_error: 0.1489
Epoch 99/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.5065 - hour_accuracy: 0.9997 - minute_accuracy: 0.8996 - real_time_error: 0.1199
Epoch 00099: saving model to D:/study/models4/model-sgd-99.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.5063 - hour_accuracy: 0.9997 - minute_accuracy: 0.8996 - real_time_error: 0.1199 - val_loss: 6.1028 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8581 - val_real_time_error: 0.1514
Epoch 100/100
14376/14400 [============================>.] - ETA: 0s - loss: 5.4896 - hour_accuracy: 0.9999 - minute_accuracy: 0.8992 - real_time_error: 0.1139
Epoch 00100: saving model to D:/study/models4/model-sgd-100.hdf5
14400/14400 [==============================] - 107s 7ms/sample - loss: 5.4898 - hour_accuracy: 0.9999 - minute_accuracy: 0.8991 - real_time_error: 0.1140 - val_loss: 6.1010 - val_hour_accuracy: 0.9967 - val_minute_accuracy: 0.8575 - val_real_time_error: 0.1517

pred_train = model.predict(X_train)
pred_test = model.predict(X_test)

print("Mean training error:", np.mean(real_time_error(y_train, pred_train)))
print("Mean test error:", np.mean(real_time_error(y_test, pred_test)))
max_error_idx = np.argmax(real_time_error(y_test, pred_test))
print("Largest test error: {} (at {})".format(np.max(real_time_error(y_test, pred_test)), max_error_idx))
print(decode_onehot(np.array([pred_test[max_error_idx]])), decode_onehot(np.array([y_test[max_error_idx]])))
plt.imsave('hardest_clock.png', np.squeeze(X_test[max_error_idx]), cmap='gray')

Mean training error: 0.04659722222222222
Mean test error: 0.15166666666666667
Largest test error: 3.0 (at 567)
[413] [416]

plt.plot(hist1.history['loss'] + hist2.history['loss'])
plt.plot(hist1.history['val_loss'] + hist2.history['val_loss'])
plt.title('Training progress')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper right')
plt.ylim(0,50)
plt.savefig('resnet_v2_adam_100_sgd_100epoch_loss.pdf')
plt.show()

plt.plot(hist1.history['hour_accuracy'] + hist2.history['hour_accuracy'])
plt.plot(hist1.history['val_hour_accuracy'] + hist2.history['val_hour_accuracy'])
plt.title('Training progress')
plt.ylabel('Hour accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper right')
plt.ylim(0,1)
plt.savefig('resnet_v2_adam_100_sgd_100epoch_hour_accuracy.pdf')
plt.show()

plt.plot(hist1.history['minute_accuracy'] + hist2.history['minute_accuracy'])
plt.plot(hist1.history['val_minute_accuracy'] + hist2.history['val_minute_accuracy'])
plt.title('Training progress')
plt.ylabel('Minute accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper right')
plt.ylim(0,1)
plt.savefig('resnet_v2_adam_100_sgd_100epoch_minute_accuracy.pdf')
plt.show()

plt.plot(hist1.history['real_time_error'] + hist2.history['real_time_error'])
plt.plot(hist1.history['val_real_time_error'] + hist2.history['val_real_time_error'])
plt.title('Training progress')
plt.ylabel('Mean absolute error (minutes)')
plt.xlabel('Epoch')
plt.ylim(0,5)
plt.legend(['Train', 'Test'], loc='upper right')
plt.savefig('resnet_v2_adam_100_sgd_100epoch_real_time.pdf')
plt.show()

layer_outputs = [layer.output for layer in model.layers] 
activation_model = keras.Model(inputs=model.input, outputs=layer_outputs)
layer_activations = activation_model.predict(X_test[0:5])

def plot_figures(figures, nrows = 1, ncols=1):
    """Plot a dictionary of figures.

    Parameters
    ----------
    figures : <title, figure> dictionary
    ncols : number of columns of subplots wanted in the display
    nrows : number of rows of subplots wanted in the figure
    """

    fig, axeslist = plt.subplots(figsize=(32,32),ncols=ncols, nrows=nrows)
    for ind,title in zip(range(len(figures)), figures):
        axeslist.ravel()[ind].imshow(figures[title], cmap='viridis')
        axeslist.ravel()[ind].set_axis_off()
        axeslist.ravel()[ind].set_xticklabels([])
        axeslist.ravel()[ind].set_yticklabels([])
        axeslist.ravel()[ind].set_aspect('equal')
    fig.subplots_adjust(hspace=0, wspace=0)
    #plt.tight_layout() # optional

plot_figures({i: layer_activations[4][0, :, :, i] for i in range(64)}, 8, 8)

tf.keras.utils.plot_model(model)

Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.